//! Hash engine (replaces the 0.1.0 `hashWorker.js`): one sequential read for every streaming //! algorithm, then the random-access City/Farm hashes over a cached view of the same file. //! Runs on a blocking thread; cancellation is checked between chunks and cache refills. use super::algorithms::Request; use super::city::{self, View}; use super::portable::{Fnv, Murmur128x64, Murmur128x86, Murmur32, SipHash24}; use crate::core::error::{CoreError, CoreResult}; use digest::{DynDigest, ExtendableOutput, Update, XofReader}; use highway::HighwayHash; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; static CRC16: crc::Crc = crc::Crc::::new(&crc::CRC_16_IBM_3740); static CRC32: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); static CRC32C: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISCSI); static CRC64_ECMA: crc::Crc = crc::Crc::::new(&crc::CRC_64_ECMA_182); static CRC64_XZ: crc::Crc = crc::Crc::::new(&crc::CRC_64_XZ); pub fn cancelled() -> CoreError { CoreError::new("HASH_CANCELLED", "Hash job was cancelled.") } fn changed(message: &str) -> CoreError { CoreError::new("HASH_FILE_CHANGED", message) } enum Stream { Digest(Box), Shake128(sha3::Shake128, usize), Shake256(sha3::Shake256, usize), K12(Box>, usize), Blake3(Box), Xxh32(xxhash_rust::xxh32::Xxh32), Xxh64(xxhash_rust::xxh64::Xxh64), Xxh3(Box, bool), Highway(highway::HighwayHasher, u32), Crc16(crc::Digest<'static, u16>), Crc32(crc::Digest<'static, u32>), Crc64(crc::Digest<'static, u64>), Adler(adler::Adler32), Murmur32(Murmur32), Murmur128x86(Murmur128x86), Murmur128x64(Murmur128x64), Sip(SipHash24), Fnv(Fnv), } impl Stream { fn new(request: &Request) -> Option { let seed = request.seed.unwrap_or(0); let digest = |d: Box| Some(Stream::Digest(d)); match request.id { "sha224" => digest(Box::new(sha2::Sha224::default())), "sha256" => digest(Box::new(sha2::Sha256::default())), "sha384" => digest(Box::new(sha2::Sha384::default())), "sha512" => digest(Box::new(sha2::Sha512::default())), "sha512-224" => digest(Box::new(sha2::Sha512_224::default())), "sha512-256" => digest(Box::new(sha2::Sha512_256::default())), "sha3-224" => digest(Box::new(sha3::Sha3_224::default())), "sha3-256" => digest(Box::new(sha3::Sha3_256::default())), "sha3-384" => digest(Box::new(sha3::Sha3_384::default())), "sha3-512" => digest(Box::new(sha3::Sha3_512::default())), "blake2b-512" => digest(Box::new(blake2::Blake2b512::default())), "blake2s-256" => digest(Box::new(blake2::Blake2s256::default())), "ripemd160" => digest(Box::new(ripemd::Ripemd160::default())), "whirlpool" => digest(Box::new(whirlpool::Whirlpool::default())), "tiger" => digest(Box::new(tiger::Tiger::default())), "tiger2" => digest(Box::new(tiger::Tiger2::default())), "md5" => digest(Box::new(md5::Md5::default())), "sha1" => digest(Box::new(sha1::Sha1::default())), "shake128" => Some(Stream::Shake128(Default::default(), request.output_bytes.unwrap_or(32))), "shake256" => Some(Stream::Shake256(Default::default(), request.output_bytes.unwrap_or(64))), "kangaroo-twelve" => Some(Stream::K12(Box::new(k12::KangarooTwelve::default()), request.output_bytes.unwrap_or(32))), "blake3" => Some(Stream::Blake3(Box::new(blake3::Hasher::new()))), "xxh32" => Some(Stream::Xxh32(xxhash_rust::xxh32::Xxh32::new(seed as u32))), "xxh64" => Some(Stream::Xxh64(xxhash_rust::xxh64::Xxh64::new(seed))), "xxh3-64" => Some(Stream::Xxh3(Box::new(xxhash_rust::xxh3::Xxh3::with_seed(seed)), false)), "xxh3-128" => Some(Stream::Xxh3(Box::new(xxhash_rust::xxh3::Xxh3::with_seed(seed)), true)), "highwayhash64" | "highwayhash128" | "highwayhash256" => { let key = request.key.as_deref().unwrap_or(&[0; 32]); let words: [u64; 4] = std::array::from_fn(|i| u64::from_le_bytes(key[i * 8..i * 8 + 8].try_into().unwrap())); let bits = match request.id { "highwayhash64" => 64, "highwayhash128" => 128, _ => 256 }; Some(Stream::Highway(highway::HighwayHasher::new(highway::Key(words)), bits)) } "crc16-ccitt-false" => Some(Stream::Crc16(CRC16.digest())), "crc32" => Some(Stream::Crc32(CRC32.digest())), "crc32c" => Some(Stream::Crc32(CRC32C.digest())), "crc64-ecma" => Some(Stream::Crc64(CRC64_ECMA.digest())), "crc64-xz" => Some(Stream::Crc64(CRC64_XZ.digest())), "adler32" => Some(Stream::Adler(adler::Adler32::new())), "murmur3-x86-32" => Some(Stream::Murmur32(Murmur32::new(seed as u32))), "murmur3-x86-128" => Some(Stream::Murmur128x86(Murmur128x86::new(seed as u32))), "murmur3-x64-128" => Some(Stream::Murmur128x64(Murmur128x64::new(seed))), "siphash-2-4" => Some(Stream::Sip(SipHash24::new(request.key.as_deref().unwrap_or(&[0; 16])))), "fnv1-32" => Some(Stream::Fnv(Fnv::new(32, false))), "fnv1-64" => Some(Stream::Fnv(Fnv::new(64, false))), "fnv1a-32" => Some(Stream::Fnv(Fnv::new(32, true))), "fnv1a-64" => Some(Stream::Fnv(Fnv::new(64, true))), _ => None, } } fn update(&mut self, data: &[u8]) { match self { Stream::Digest(d) => d.update(data), Stream::Shake128(h, _) => Update::update(h, data), Stream::Shake256(h, _) => Update::update(h, data), Stream::K12(h, _) => Update::update(h.as_mut(), data), Stream::Blake3(h) => { h.update(data); } Stream::Xxh32(h) => h.update(data), Stream::Xxh64(h) => h.update(data), Stream::Xxh3(h, _) => h.update(data), Stream::Highway(h, _) => h.append(data), Stream::Crc16(h) => h.update(data), Stream::Crc32(h) => h.update(data), Stream::Crc64(h) => h.update(data), Stream::Adler(h) => h.write_slice(data), Stream::Murmur32(h) => h.update(data), Stream::Murmur128x86(h) => h.update(data), Stream::Murmur128x64(h) => h.update(data), Stream::Sip(h) => h.update(data), Stream::Fnv(h) => h.update(data), } } fn finish(self) -> Vec { fn xof(reader: &mut dyn XofReader, n: usize) -> Vec { let mut out = vec![0u8; n]; reader.read(&mut out); out } match self { Stream::Digest(d) => d.finalize().to_vec(), Stream::Shake128(h, n) => xof(&mut h.finalize_xof(), n), Stream::Shake256(h, n) => xof(&mut h.finalize_xof(), n), Stream::K12(h, n) => xof(&mut h.finalize_xof(), n), Stream::Blake3(h) => h.finalize().as_bytes().to_vec(), Stream::Xxh32(h) => h.digest().to_be_bytes().to_vec(), Stream::Xxh64(h) => h.digest().to_be_bytes().to_vec(), Stream::Xxh3(h, wide) => if wide { h.digest128().to_be_bytes().to_vec() } else { h.digest().to_be_bytes().to_vec() }, Stream::Highway(h, bits) => match bits { 64 => h.finalize64().to_le_bytes().to_vec(), // 0.1.0 (highwayhasher) byte order 128 => h.finalize128().iter().flat_map(|w| w.to_le_bytes()).collect(), _ => h.finalize256().iter().flat_map(|w| w.to_le_bytes()).collect(), }, Stream::Crc16(h) => h.finalize().to_be_bytes().to_vec(), Stream::Crc32(h) => h.finalize().to_be_bytes().to_vec(), Stream::Crc64(h) => h.finalize().to_be_bytes().to_vec(), Stream::Adler(h) => h.checksum().to_be_bytes().to_vec(), Stream::Murmur32(h) => h.finish(), Stream::Murmur128x86(h) => h.finish(), Stream::Murmur128x64(h) => h.finish(), Stream::Sip(h) => h.finish(), Stream::Fnv(h) => h.finish(), } } } fn is_random_access(id: &str) -> bool { matches!(id, "cityhash32" | "cityhash64" | "cityhash128" | "farmhash32" | "farmhash64" | "farmhash128") } fn random_access(requests: &[Request], view: &mut dyn View) -> Vec<(&'static str, Vec)> { let mut out = Vec::new(); let mut wide: Option> = None; for request in requests.iter().filter(|r| is_random_access(r.id)) { let digest = match request.id { "cityhash32" => city::city32(view).to_be_bytes().to_vec(), "cityhash64" => city::city64(view).to_be_bytes().to_vec(), "farmhash32" => city::farm32(view).to_be_bytes().to_vec(), "farmhash64" => city::farm64(view).to_be_bytes().to_vec(), // FarmHash Fingerprint128 is CityHash128 by definition (0.1.0 computed it once). _ => wide.get_or_insert_with(|| city::digest128(city::city128(view))).clone(), }; out.push((request.id, digest)); } out } /// Pure in-memory digests (tests, small inputs). pub fn digest_bytes(requests: &[Request], data: &[u8]) -> Vec<(&'static str, Vec)> { let mut streams: Vec<(&'static str, Stream)> = requests.iter().filter_map(|r| Stream::new(r).map(|s| (r.id, s))).collect(); for (_, stream) in streams.iter_mut() { stream.update(data); } let mut results: Vec<(&'static str, Vec)> = streams.into_iter().map(|(id, s)| (id, s.finish())).collect(); results.extend(random_access(requests, &mut city::SliceView(data))); order(requests, results) } fn order(requests: &[Request], results: Vec<(&'static str, Vec)>) -> Vec<(&'static str, Vec)> { requests.iter().filter_map(|r| results.iter().find(|(id, _)| *id == r.id).cloned()).collect() } /// File identity captured when a file is authorized (0.1.0: dev, ino, size, mtimeNs). #[derive(Debug, Clone, PartialEq, Eq)] pub struct Identity { pub size: u64, pub modified_ns: i128, pub device: u64, pub inode: u64, } impl Identity { pub fn of(meta: &std::fs::Metadata) -> Identity { let modified_ns = meta .modified() .ok() .map(|t| match t.duration_since(std::time::UNIX_EPOCH) { Ok(d) => d.as_nanos() as i128, Err(e) => -(e.duration().as_nanos() as i128), }) .unwrap_or(0); #[cfg(unix)] let (device, inode) = { use std::os::unix::fs::MetadataExt; (meta.dev(), meta.ino()) }; #[cfg(windows)] let (device, inode) = { use std::os::windows::fs::MetadataExt; (0u64, meta.creation_time()) }; Identity { size: meta.len(), modified_ns, device, inode } } } /// Opens a regular file without following a final symbolic link. pub fn open_no_follow(path: &Path) -> std::io::Result { let mut options = std::fs::OpenOptions::new(); options.read(true); #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW); } #[cfg(windows)] { use std::os::windows::fs::OpenOptionsExt; options.custom_flags(0x0020_0000); // FILE_FLAG_OPEN_REPARSE_POINT } options.open(path) } /// Lazily cached file view for City/Farm (reads 1 MiB windows; checks cancellation). struct FileView<'a> { file: &'a mut File, len: usize, cache: Vec, start: usize, cancel: &'a AtomicBool, error: Option, } const WINDOW: usize = 1024 * 1024; impl FileView<'_> { fn ensure(&mut self, offset: usize, width: usize) -> Option { if self.error.is_some() { return None; } if offset >= self.start && offset + width <= self.start + self.cache.len() { return Some(offset - self.start); } if self.cancel.load(Ordering::Relaxed) { self.error = Some(cancelled()); return None; } let start = (offset / WINDOW) * WINDOW; let bytes = (WINDOW + 16).min(self.len - start); self.cache.resize(bytes, 0); let result = self.file.seek(SeekFrom::Start(start as u64)).and_then(|_| self.file.read_exact(&mut self.cache)); if result.is_err() { self.error = Some(changed("Selected file ended unexpectedly.")); self.cache.clear(); return None; } self.start = start; Some(offset - start) } } impl View for FileView<'_> { fn len(&self) -> usize { self.len } fn byte(&mut self, offset: usize) -> u8 { self.ensure(offset, 1).map(|i| self.cache[i]).unwrap_or(0) } fn u32(&mut self, offset: usize) -> u32 { self.ensure(offset, 4).map(|i| u32::from_le_bytes(self.cache[i..i + 4].try_into().unwrap())).unwrap_or(0) } fn u64(&mut self, offset: usize) -> u64 { self.ensure(offset, 8).map(|i| u64::from_le_bytes(self.cache[i..i + 8].try_into().unwrap())).unwrap_or(0) } } pub const CHUNK: usize = 1024 * 1024; /// Hashes one file. `progress(bytes_read)` is called at most every 100 ms and at the end. pub fn hash_file(path: &Path, expected: &Identity, requests: &[Request], cancel: &AtomicBool, mut progress: impl FnMut(u64)) -> CoreResult)>> { let mut file = open_no_follow(path).map_err(super::service::io_error)?; let before = file.metadata().map_err(super::service::io_error)?; if !before.is_file() { return Err(CoreError::new("HASH_NOT_REGULAR_FILE", "Only regular files can be hashed.")); } if Identity::of(&before) != *expected { return Err(changed("The selected file changed before hashing started.")); } let size = before.len(); let mut streams: Vec<(&'static str, Stream)> = requests.iter().filter(|r| !is_random_access(r.id)).filter_map(|r| Stream::new(r).map(|s| (r.id, s))).collect(); let mut buffer = vec![0u8; CHUNK]; let mut position = 0u64; let mut last = std::time::Instant::now() - std::time::Duration::from_secs(1); while position < size { if cancel.load(Ordering::Relaxed) { return Err(cancelled()); } let want = ((size - position) as usize).min(buffer.len()); let read = file.read(&mut buffer[..want]).map_err(super::service::io_error)?; if read == 0 { return Err(changed("Selected file ended unexpectedly.")); } for (_, stream) in streams.iter_mut() { stream.update(&buffer[..read]); } position += read as u64; if last.elapsed() >= std::time::Duration::from_millis(100) || position == size { progress(position); last = std::time::Instant::now(); } } if size == 0 { progress(0); } let mut results: Vec<(&'static str, Vec)> = streams.into_iter().map(|(id, s)| (id, s.finish())).collect(); if requests.iter().any(|r| is_random_access(r.id)) { if size > usize::MAX as u64 { return Err(CoreError::new("HASH_FILE_TOO_LARGE", "Selected file exceeds the supported size.")); } let mut view = FileView { file: &mut file, len: size as usize, cache: Vec::new(), start: 0, cancel, error: None }; let digests = random_access(requests, &mut view); if let Some(error) = view.error.take() { return Err(error); } results.extend(digests); } if cancel.load(Ordering::Relaxed) { return Err(cancelled()); } let after = file.metadata().map_err(super::service::io_error)?; if Identity::of(&after) != *expected { return Err(changed("The file changed while it was being hashed.")); } Ok(order(requests, results)) } #[cfg(test)] mod tests { use super::super::algorithms::{normalize_requests, ALGORITHMS}; use super::*; use serde_json::{json, Value}; fn data(n: usize) -> Vec { (0..n).map(|i| (((i * 31 + 7) ^ (i >> 8) ^ (i >> 3)) & 0xff) as u8).collect() } /// Bit-exact parity with the 0.1.0 Electron worker (vectors generated from its code). #[test] fn parity_with_0_1_0_worker() { let fixture: Value = serde_json::from_str(include_str!("../../../tests/fixtures/hash/parity-0.1.0.json")).unwrap(); let variants: Vec> = fixture["variants"].as_array().unwrap().iter().map(|v| normalize_requests(v).unwrap()).collect(); let mut failures: Vec = Vec::new(); for case in fixture["cases"].as_array().unwrap() { let length = case["length"].as_u64().unwrap() as usize; let requests = &variants[case["variant"].as_u64().unwrap() as usize]; let input = data(length); for (id, digest) in digest_bytes(requests, &input) { // 0.1.0 MurmurHash3 x86-128 mixed the block state in the wrong order (non-standard // for inputs of 16+ bytes); 0.1.1 uses the reference algorithm (see test below). if id == "murmur3-x86-128" && length >= 16 { continue; } let expected = case["digests"][id].as_str().unwrap(); if hex::encode(&digest) != expected { failures.push(format!("{id} len={length} variant={}", case["variant"])); } } } let mut by_algorithm: std::collections::BTreeMap = Default::default(); for f in &failures { *by_algorithm.entry(f.split(' ').next().unwrap().to_string()).or_default() += 1; } assert!(failures.is_empty(), "{} mismatches: {:?}; first: {:?}", failures.len(), by_algorithm, failures.iter().take(5).collect::>()); } /// Published reference vectors (FIPS/RFC/upstream suites) shipped since 0.1.0. #[test] fn standard_vectors() { let fixture: Value = serde_json::from_str(include_str!("../../../../tools/test-fixtures/hash-vectors.json")).unwrap(); let mut failures = Vec::new(); for (name, message) in fixture["messages"].as_object().unwrap() { let vectors = fixture["vectors"][name].as_object().unwrap(); let ids: Vec = vectors.keys().filter(|id| super::super::algorithms::by_id(id).is_some()).map(|id| json!(id)).collect(); let mut requests = normalize_requests(&Value::Array(ids)).unwrap(); for r in requests.iter_mut() { if r.key.is_some() { let n = r.key.as_ref().unwrap().len(); r.key = Some((0..n as u8).collect()); } } for (id, digest) in digest_bytes(&requests, message.as_str().unwrap().as_bytes()) { if hex::encode(&digest) != vectors[id].as_str().unwrap() { failures.push(format!("{id}/{name}")); } } } assert!(failures.is_empty(), "{failures:?}"); } /// MurmurHash3 x86-128 against the independent `murmur3` crate (reference C++ port). #[test] fn murmur_x86_128_matches_reference() { for (length, seed) in [(0usize, 0u32), (3, 7), (15, 0), (16, 0), (17, 1), (40, 0x9e3779b9), (1000, 42)] { let input = data(length); let reference = murmur3::murmur3_x86_128(&mut std::io::Cursor::new(&input), seed).unwrap(); let mut ours = Murmur128x86::new(seed); ours.update(&input); // The crate packs h1 into the low word; 0.1.0/0.1.1 print h1..h4 big-endian. let words = [reference as u32, (reference >> 32) as u32, (reference >> 64) as u32, (reference >> 96) as u32]; let expected: Vec = words.iter().flat_map(|w| w.to_be_bytes()).collect(); assert_eq!(ours.finish(), expected, "length {length}"); } } #[test] fn file_hash_matches_memory_and_detects_cancel() { let dir = tempfile::tempdir().unwrap(); let path = dir.path().join("f.bin"); let input = data(3 * CHUNK + 12345); std::fs::write(&path, &input).unwrap(); let all: Vec = ALGORITHMS.iter().map(|a| json!(a.id)).collect(); let requests = normalize_requests(&Value::Array(all)).unwrap(); let identity = Identity::of(&std::fs::metadata(&path).unwrap()); let cancel = AtomicBool::new(false); let mut seen = 0; let from_file = hash_file(&path, &identity, &requests, &cancel, |b| seen = b).unwrap(); assert_eq!(seen, input.len() as u64); assert_eq!(from_file, digest_bytes(&requests, &input)); cancel.store(true, Ordering::Relaxed); assert_eq!(hash_file(&path, &identity, &requests, &cancel, |_| {}).unwrap_err().code, "HASH_CANCELLED"); let stale = Identity { size: 1, ..identity }; assert_eq!(hash_file(&path, &stale, &requests, &AtomicBool::new(false), |_| {}).unwrap_err().code, "HASH_FILE_CHANGED"); } }