//! CityHash v1.1 (32/64/128) and FarmHash fingerprints (farmhashmk::Hash32, //! farmhashna::Hash64) over a random-access view, ported from the 0.1.0 code //! (`hashing/cityHash.js`, google/cityhash, MIT) and farmhashjs (Apache-2.0). //! The view reads the file lazily so arbitrarily large files never sit in memory. const K0: u64 = 0xc3a5c85c97cb3127; const K1: u64 = 0xb492b66fbe98f273; const K2: u64 = 0x9ae16a3b2f90404f; const C1: u32 = 0xcc9e2d51; const C2: u32 = 0x1b873593; /// Little-endian random access to the hashed input. pub trait View { fn len(&self) -> usize; fn byte(&mut self, offset: usize) -> u8; fn u32(&mut self, offset: usize) -> u32; fn u64(&mut self, offset: usize) -> u64; } pub struct SliceView<'a>(pub &'a [u8]); impl View for SliceView<'_> { fn len(&self) -> usize { self.0.len() } fn byte(&mut self, offset: usize) -> u8 { self.0[offset] } fn u32(&mut self, offset: usize) -> u32 { u32::from_le_bytes(self.0[offset..offset + 4].try_into().unwrap()) } fn u64(&mut self, offset: usize) -> u64 { u64::from_le_bytes(self.0[offset..offset + 8].try_into().unwrap()) } } #[inline] fn rot32(v: u32, s: u32) -> u32 { v.rotate_right(s) } #[inline] fn rot64(v: u64, s: u32) -> u64 { v.rotate_right(s) } #[inline] fn fmix(mut h: u32) -> u32 { h ^= h >> 16; h = h.wrapping_mul(0x85ebca6b); h ^= h >> 13; h = h.wrapping_mul(0xc2b2ae35); h ^ (h >> 16) } #[inline] fn mur(mut a: u32, mut h: u32) -> u32 { a = a.wrapping_mul(C1); a = rot32(a, 17); a = a.wrapping_mul(C2); h ^= a; h = rot32(h, 19); h.wrapping_mul(5).wrapping_add(0xe6546b64) } #[inline] fn shift_mix(v: u64) -> u64 { v ^ (v >> 47) } #[inline] fn hash_len16(u: u64, v: u64, mul: u64) -> u64 { let mut a = (u ^ v).wrapping_mul(mul); a ^= a >> 47; let mut b = (v ^ a).wrapping_mul(mul); b ^= b >> 47; b.wrapping_mul(mul) } const KMUL: u64 = 0x9ddfea08eb382d69; fn weak_hash(view: &mut dyn View, offset: usize, mut a: u64, mut b: u64) -> (u64, u64) { let w = view.u64(offset); let x = view.u64(offset + 8); let y = view.u64(offset + 16); let z = view.u64(offset + 24); a = a.wrapping_add(w); b = rot64(b.wrapping_add(a).wrapping_add(z), 21); let c = a; a = a.wrapping_add(x).wrapping_add(y); b = b.wrapping_add(rot64(a, 44)); (a.wrapping_add(z), b.wrapping_add(c)) } #[inline] fn mix20(v: u32) -> u32 { rot32(v.wrapping_mul(C1), 17).wrapping_mul(C2) } pub fn city32(view: &mut dyn View) -> u32 { let len = view.len(); let l32 = len as u32; if len <= 4 { let (mut b, mut c) = (0u32, 9u32); for i in 0..len { let signed = view.byte(i) as i8 as i32 as u32; b = b.wrapping_mul(C1).wrapping_add(signed); c ^= b; } return fmix(mur(b, mur(l32, c))); } if len <= 12 { let mut a = l32; let mut b = l32.wrapping_mul(5); let c = 9u32.wrapping_add(view.u32((len >> 1) & 4)); let d = b; a = a.wrapping_add(view.u32(0)); b = b.wrapping_add(view.u32(len - 4)); return fmix(mur(c, mur(b, mur(a, d)))); } if len <= 24 { let a = view.u32((len >> 1) - 4); let b = view.u32(4); let c = view.u32(len - 8); let d = view.u32(len >> 1); let e = view.u32(0); let f = view.u32(len - 4); return fmix(mur(f, mur(e, mur(d, mur(c, mur(b, mur(a, l32))))))); } let mut h = l32; let mut g = C1.wrapping_mul(h); let mut f = g; let t: Vec = [len - 4, len - 8, len - 16, len - 12, len - 20].iter().map(|o| mix20(view.u32(*o))).collect(); let step = |x: u32| rot32(x, 19).wrapping_mul(5).wrapping_add(0xe6546b64); h ^= t[0]; h = step(h); h ^= t[2]; h = step(h); g ^= t[1]; g = step(g); g ^= t[3]; g = step(g); f = f.wrapping_add(t[4]); f = step(f); let mut offset = 0; let mut iterations = (len - 1) / 20; loop { let a0 = mix20(view.u32(offset)); let a1 = view.u32(offset + 4); let a2 = mix20(view.u32(offset + 8)); let a3 = mix20(view.u32(offset + 12)); let a4 = view.u32(offset + 16); h ^= a0; h = rot32(h, 18).wrapping_mul(5).wrapping_add(0xe6546b64); f = f.wrapping_add(a1); f = rot32(f, 19).wrapping_mul(C1); g = g.wrapping_add(a2); g = rot32(g, 18).wrapping_mul(5).wrapping_add(0xe6546b64); h ^= a3.wrapping_add(a1); h = rot32(h, 19).wrapping_mul(5).wrapping_add(0xe6546b64); g ^= a4; g = g.swap_bytes().wrapping_mul(5); h = h.wrapping_add(a4.wrapping_mul(5)).swap_bytes(); f = f.wrapping_add(a0); let old_f = f; f = g; g = h; h = old_f; offset += 20; iterations -= 1; if iterations == 0 { break; } } g = rot32(rot32(g, 11).wrapping_mul(C1), 17).wrapping_mul(C1); f = rot32(rot32(f, 11).wrapping_mul(C1), 17).wrapping_mul(C1); h = rot32(h.wrapping_add(g), 19); h = h.wrapping_mul(5).wrapping_add(0xe6546b64); h = rot32(h, 17).wrapping_mul(C1); h = rot32(h.wrapping_add(f), 19); h = h.wrapping_mul(5).wrapping_add(0xe6546b64); rot32(h, 17).wrapping_mul(C1) } fn hash0to16(view: &mut dyn View, offset: usize, len: usize) -> u64 { if len >= 8 { let mul = K2.wrapping_add(len as u64 * 2); let a = view.u64(offset).wrapping_add(K2); let b = view.u64(offset + len - 8); let c = rot64(b, 37).wrapping_mul(mul).wrapping_add(a); let d = rot64(a, 25).wrapping_add(b).wrapping_mul(mul); return hash_len16(c, d, mul); } if len >= 4 { let mul = K2.wrapping_add(len as u64 * 2); let a = view.u32(offset) as u64; return hash_len16((len as u64).wrapping_add(a << 3), view.u32(offset + len - 4) as u64, mul); } if len > 0 { let a = view.byte(offset) as u64; let b = view.byte(offset + (len >> 1)) as u64; let c = view.byte(offset + len - 1) as u64; let y = a.wrapping_add(b << 8); let z = (len as u64).wrapping_add(c << 2); return shift_mix(y.wrapping_mul(K2) ^ z.wrapping_mul(K0)).wrapping_mul(K2); } K2 } fn hash17to32(view: &mut dyn View, offset: usize, len: usize) -> u64 { let mul = K2.wrapping_add(len as u64 * 2); let a = view.u64(offset).wrapping_mul(K1); let b = view.u64(offset + 8); let c = view.u64(offset + len - 8).wrapping_mul(mul); let d = view.u64(offset + len - 16).wrapping_mul(K2); hash_len16( rot64(a.wrapping_add(b), 43).wrapping_add(rot64(c, 30)).wrapping_add(d), a.wrapping_add(rot64(b.wrapping_add(K2), 18)).wrapping_add(c), mul, ) } fn city_hash33to64(view: &mut dyn View, offset: usize, len: usize) -> u64 { let mul = K2.wrapping_add(len as u64 * 2); let mut a = view.u64(offset).wrapping_mul(K2); let mut b = view.u64(offset + 8); let c = view.u64(offset + len - 24); let d = view.u64(offset + len - 32); let e = view.u64(offset + 16).wrapping_mul(K2); let f = view.u64(offset + 24).wrapping_mul(9); let g = view.u64(offset + len - 8); let h = view.u64(offset + len - 16).wrapping_mul(mul); let u = rot64(a.wrapping_add(g), 43).wrapping_add(rot64(b, 30).wrapping_add(c).wrapping_mul(9)); let v = (a.wrapping_add(g) ^ d).wrapping_add(f).wrapping_add(1); let w = u.wrapping_add(v).wrapping_mul(mul).swap_bytes().wrapping_add(h); let x = rot64(e.wrapping_add(f), 42).wrapping_add(c); let y = v.wrapping_add(w).wrapping_mul(mul).swap_bytes().wrapping_add(g).wrapping_mul(mul); let z = e.wrapping_add(f).wrapping_add(c); a = x.wrapping_add(z).wrapping_mul(mul).wrapping_add(y).swap_bytes().wrapping_add(b); b = shift_mix(z.wrapping_add(a).wrapping_mul(mul).wrapping_add(d).wrapping_add(h)).wrapping_mul(mul); b.wrapping_add(x) } pub fn city64(view: &mut dyn View) -> u64 { let len = view.len(); if len <= 16 { return hash0to16(view, 0, len); } if len <= 32 { return hash17to32(view, 0, len); } if len <= 64 { return city_hash33to64(view, 0, len); } let l = len as u64; let mut x = view.u64(len - 40); let mut y = view.u64(len - 16).wrapping_add(view.u64(len - 56)); let mut z = hash_len16(view.u64(len - 48).wrapping_add(l), view.u64(len - 24), KMUL); let mut v = weak_hash(view, len - 64, l, z); let mut w = weak_hash(view, len - 32, y.wrapping_add(K1), x); x = x.wrapping_mul(K1).wrapping_add(view.u64(0)); let mut remaining = (len - 1) & !63usize; let mut offset = 0usize; loop { x = rot64(x.wrapping_add(y).wrapping_add(v.0).wrapping_add(view.u64(offset + 8)), 37).wrapping_mul(K1); y = rot64(y.wrapping_add(v.1).wrapping_add(view.u64(offset + 48)), 42).wrapping_mul(K1); x ^= w.1; y = y.wrapping_add(v.0).wrapping_add(view.u64(offset + 40)); z = rot64(z.wrapping_add(w.0), 33).wrapping_mul(K1); v = weak_hash(view, offset, v.1.wrapping_mul(K1), x.wrapping_add(w.0)); let seed_b = y.wrapping_add(view.u64(offset + 16)); w = weak_hash(view, offset + 32, z.wrapping_add(w.1), seed_b); std::mem::swap(&mut z, &mut x); offset += 64; remaining -= 64; if remaining == 0 { break; } } hash_len16( hash_len16(v.0, w.0, KMUL).wrapping_add(shift_mix(y).wrapping_mul(K1)).wrapping_add(z), hash_len16(v.1, w.1, KMUL).wrapping_add(x), KMUL, ) } fn city_murmur(view: &mut dyn View, offset: usize, len: usize, seed: (u64, u64)) -> (u64, u64) { let (mut a, mut b) = seed; let mut c; let mut d; let mut position = offset; let mut remaining = len as i64; if len <= 16 { a = shift_mix(a.wrapping_mul(K1)).wrapping_mul(K1); c = b.wrapping_mul(K1).wrapping_add(hash0to16(view, position, len)); d = shift_mix(a.wrapping_add(if len >= 8 { view.u64(position) } else { c })); } else { c = hash_len16(view.u64(position + len - 8).wrapping_add(K1), a, KMUL); d = hash_len16(b.wrapping_add(len as u64), c.wrapping_add(view.u64(position + len - 16)), KMUL); a = a.wrapping_add(d); loop { a ^= shift_mix(view.u64(position).wrapping_mul(K1)).wrapping_mul(K1); a = a.wrapping_mul(K1); b ^= a; c ^= shift_mix(view.u64(position + 8).wrapping_mul(K1)).wrapping_mul(K1); c = c.wrapping_mul(K1); d ^= c; position += 16; remaining -= 16; if remaining <= 16 { break; } } } a = hash_len16(a, c, KMUL); b = hash_len16(d, b, KMUL); (a ^ b, hash_len16(b, a, KMUL)) } fn city128_with_seed(view: &mut dyn View, base: usize, len: usize, seed: (u64, u64)) -> (u64, u64) { if len < 128 { return city_murmur(view, base, len, seed); } let (mut x, mut y) = seed; let mut z = (len as u64).wrapping_mul(K1); let mut offset = base; let mut remaining = len; let mut v = (0u64, 0u64); v.0 = rot64(y ^ K1, 49).wrapping_mul(K1).wrapping_add(view.u64(offset)); v.1 = rot64(v.0, 42).wrapping_mul(K1).wrapping_add(view.u64(offset + 8)); let mut w = (rot64(y.wrapping_add(z), 35).wrapping_mul(K1).wrapping_add(x), rot64(x.wrapping_add(view.u64(offset + 88)), 53).wrapping_mul(K1)); loop { for _ in 0..2 { x = rot64(x.wrapping_add(y).wrapping_add(v.0).wrapping_add(view.u64(offset + 8)), 37).wrapping_mul(K1); y = rot64(y.wrapping_add(v.1).wrapping_add(view.u64(offset + 48)), 42).wrapping_mul(K1); x ^= w.1; y = y.wrapping_add(v.0).wrapping_add(view.u64(offset + 40)); z = rot64(z.wrapping_add(w.0), 33).wrapping_mul(K1); v = weak_hash(view, offset, v.1.wrapping_mul(K1), x.wrapping_add(w.0)); let seed_b = y.wrapping_add(view.u64(offset + 16)); w = weak_hash(view, offset + 32, z.wrapping_add(w.1), seed_b); std::mem::swap(&mut z, &mut x); offset += 64; } remaining -= 128; if remaining < 128 { break; } } x = x.wrapping_add(rot64(v.0.wrapping_add(z), 49).wrapping_mul(K0)); y = y.wrapping_mul(K0).wrapping_add(rot64(w.1, 37)); z = z.wrapping_mul(K0).wrapping_add(rot64(w.0, 27)); w.0 = w.0.wrapping_mul(9); v.0 = v.0.wrapping_mul(K0); let mut done = 0; while done < remaining { done += 32; y = rot64(x.wrapping_add(y), 42).wrapping_mul(K0).wrapping_add(v.1); w.0 = w.0.wrapping_add(view.u64(offset + remaining - done + 16)); x = x.wrapping_mul(K0).wrapping_add(w.0); z = z.wrapping_add(w.1).wrapping_add(view.u64(offset + remaining - done)); w.1 = w.1.wrapping_add(v.0); v = weak_hash(view, offset + remaining - done, v.0.wrapping_add(z), v.1); v.0 = v.0.wrapping_mul(K0); } x = hash_len16(x, v.0, KMUL); y = hash_len16(y.wrapping_add(z), w.0, KMUL); (hash_len16(x.wrapping_add(v.1), w.1, KMUL).wrapping_add(y), hash_len16(x.wrapping_add(w.1), y.wrapping_add(v.1), KMUL)) } pub fn city128(view: &mut dyn View) -> (u64, u64) { let len = view.len(); if len >= 16 { let seed = (view.u64(0), view.u64(8).wrapping_add(K0)); city128_with_seed(view, 16, len - 16, seed) } else { city128_with_seed(view, 0, len, (K0, K1)) } } /// 0.1.0 digest layout: high (second) word first, both big-endian. pub fn digest128(value: (u64, u64)) -> Vec { [value.1.to_be_bytes(), value.0.to_be_bytes()].concat() } // --- FarmHash fingerprints ---------------------------------------------------------------------- fn farm_mk_hash32(view: &mut dyn View) -> u32 { let len = view.len(); let l32 = len as u32; if len <= 4 { return city32(view); // identical algorithm for 0..4 bytes (seed 0) } if len <= 12 { let mut a = l32; let mut b = l32.wrapping_mul(5); let mut c = 9u32; let d = b; a = a.wrapping_add(view.u32(0)); b = b.wrapping_add(view.u32(len - 4)); c = c.wrapping_add(view.u32((len >> 1) & 4)); return fmix(mur(c, mur(b, mur(a, d)))); } if len <= 24 { let a = view.u32((len >> 1) - 4); let b = view.u32(4); let c = view.u32(len - 8); let d = view.u32(len >> 1); let e = view.u32(0); let f = view.u32(len - 4); let mut h = d.wrapping_mul(C1).wrapping_add(l32); let mut a2 = rot32(a, 12).wrapping_add(f); h = mur(c, h).wrapping_add(a2); a2 = rot32(a2, 3).wrapping_add(c); h = mur(e, h).wrapping_add(a2); a2 = rot32(a2.wrapping_add(f), 12).wrapping_add(d); h = mur(b, h).wrapping_add(a2); return fmix(h); } let mut h = l32; let mut g = C1.wrapping_mul(l32); let mut f = g; let a0 = mix20(view.u32(len - 4)); let a1 = mix20(view.u32(len - 8)); let a2 = mix20(view.u32(len - 16)); let a3 = mix20(view.u32(len - 12)); let a4 = mix20(view.u32(len - 20)); let step = |x: u32| rot32(x, 19).wrapping_mul(5).wrapping_add(0xe6546b64); h ^= a0; h = step(h); h ^= a2; h = step(h); g ^= a1; g = step(g); g ^= a3; g = step(g); f = f.wrapping_add(a4); f = rot32(f, 19).wrapping_add(113); let mut iterations = (len - 1) / 20; let mut offset = 0; loop { let a = view.u32(offset); let b = view.u32(offset + 4); let c = view.u32(offset + 8); let d = view.u32(offset + 12); let e = view.u32(offset + 16); h = h.wrapping_add(a); g = g.wrapping_add(b); f = f.wrapping_add(c); h = mur(d, h).wrapping_add(e); g = mur(c, g).wrapping_add(a); f = mur(b.wrapping_add(e.wrapping_mul(C1)), f).wrapping_add(d); f = f.wrapping_add(g); g = g.wrapping_add(f); offset += 20; iterations -= 1; if iterations == 0 { break; } } g = rot32(rot32(g, 11).wrapping_mul(C1), 17).wrapping_mul(C1); f = rot32(rot32(f, 11).wrapping_mul(C1), 17).wrapping_mul(C1); h = rot32(h.wrapping_add(g), 19); h = h.wrapping_mul(5).wrapping_add(0xe6546b64); h = rot32(h, 17).wrapping_mul(C1); h = rot32(h.wrapping_add(f), 19); h = h.wrapping_mul(5).wrapping_add(0xe6546b64); rot32(h, 17).wrapping_mul(C1) } pub fn farm32(view: &mut dyn View) -> u32 { farm_mk_hash32(view) } fn na_hash33to64(view: &mut dyn View, len: usize) -> u64 { let mul = K2.wrapping_add(len as u64 * 2); let a = view.u64(0).wrapping_mul(K2); let b = view.u64(8); let c = view.u64(len - 8).wrapping_mul(mul); let d = view.u64(len - 16).wrapping_mul(K2); let y = rot64(a.wrapping_add(b), 43).wrapping_add(rot64(c, 30)).wrapping_add(d); let z = hash_len16(y, a.wrapping_add(rot64(b.wrapping_add(K2), 18)).wrapping_add(c), mul); let e = view.u64(16).wrapping_mul(mul); let f = view.u64(24); let g = y.wrapping_add(view.u64(len - 32)).wrapping_mul(mul); let h = z.wrapping_add(view.u64(len - 24)).wrapping_mul(mul); hash_len16(rot64(e.wrapping_add(f), 43).wrapping_add(rot64(g, 30)).wrapping_add(h), e.wrapping_add(rot64(f.wrapping_add(a), 18)).wrapping_add(g), mul) } pub fn farm64(view: &mut dyn View) -> u64 { let len = view.len(); if len <= 16 { return hash0to16(view, 0, len); } if len <= 32 { return hash17to32(view, 0, len); } if len <= 64 { return na_hash33to64(view, len); } let seed = 81u64; let mut x = seed; let mut y = seed.wrapping_mul(K1).wrapping_add(113); let mut z = shift_mix(y.wrapping_mul(K2).wrapping_add(113)).wrapping_mul(K2); let mut v = (0u64, 0u64); let mut w = (0u64, 0u64); x = x.wrapping_mul(K2).wrapping_add(view.u64(0)); let end = ((len - 1) / 64) * 64; let last64 = end + ((len - 1) & 63) - 63; let mut idx = 0; loop { x = rot64(x.wrapping_add(y).wrapping_add(v.0).wrapping_add(view.u64(idx + 8)), 37).wrapping_mul(K1); y = rot64(y.wrapping_add(v.1).wrapping_add(view.u64(idx + 48)), 42).wrapping_mul(K1); x ^= w.1; y = y.wrapping_add(v.0).wrapping_add(view.u64(idx + 40)); z = rot64(z.wrapping_add(w.0), 33).wrapping_mul(K1); v = weak_hash(view, idx, v.1.wrapping_mul(K1), x.wrapping_add(w.0)); let seed_b = y.wrapping_add(view.u64(idx + 16)); w = weak_hash(view, idx + 32, z.wrapping_add(w.1), seed_b); std::mem::swap(&mut z, &mut x); idx += 64; if idx == end { break; } } let mul = K1.wrapping_add((z & 0xff) << 1); idx = last64; w.0 = w.0.wrapping_add(((len - 1) & 63) as u64); v.0 = v.0.wrapping_add(w.0); w.0 = w.0.wrapping_add(v.0); x = rot64(x.wrapping_add(y).wrapping_add(v.0).wrapping_add(view.u64(idx + 8)), 37).wrapping_mul(mul); y = rot64(y.wrapping_add(v.1).wrapping_add(view.u64(idx + 48)), 42).wrapping_mul(mul); x ^= w.1.wrapping_mul(9); y = y.wrapping_add(v.0.wrapping_mul(9)).wrapping_add(view.u64(idx + 40)); z = rot64(z.wrapping_add(w.0), 33).wrapping_mul(mul); v = weak_hash(view, idx, v.1.wrapping_mul(mul), x.wrapping_add(w.0)); let seed_b = y.wrapping_add(view.u64(idx + 16)); w = weak_hash(view, idx + 32, z.wrapping_add(w.1), seed_b); std::mem::swap(&mut z, &mut x); hash_len16( hash_len16(v.0, w.0, mul).wrapping_add(shift_mix(y).wrapping_mul(K0)).wrapping_add(z), hash_len16(v.1, w.1, mul).wrapping_add(x), mul, ) }