Skip to content
Browse files

Switch MD5 with City

  • Loading branch information...
1 parent cc04067 commit 82a791d89dfe85d1f4a77b7bc68eb3ba90857571 @vmg committed Aug 4, 2012
Showing with 640 additions and 422 deletions.
  1. +1 −1 Makefile
  2. +519 −0 src/city.c
  3. +86 −0 src/city.h
  4. +30 −47 src/dablooms.c
  5. +4 −3 src/dablooms.h
  6. +0 −332 src/md5.c
  7. +0 −39 src/md5.h
View
2 Makefile
@@ -59,7 +59,7 @@ PY_MOD_DIR := $(shell $(PYTHON) -c "import distutils.sysconfig ; print(distutils
PY_FLAGS = --build-lib=$(PY_BLDDIR) --build-temp=$(PY_BLDDIR)
PY_BLD_ENV = INCPATH="$(SRCDIR) $(INCPATH)" LIBPATH="$(BLDDIR) $(LIBPATH)"
-SRCS_LIBDABLOOMS = md5.c dablooms.c
+SRCS_LIBDABLOOMS = city.c dablooms.c
SRCS_TESTS = test_dablooms.c
WORDS = /usr/share/dict/words
OBJS_LIBDABLOOMS = $(patsubst %.c, $(BLDDIR)/%.o, $(SRCS_LIBDABLOOMS))
View
519 src/city.c
@@ -0,0 +1,519 @@
+// city.c - cityhash-c
+// CityHash on C
+// Copyright (c) 2011-2012, Alexander Nusov
+//
+// - original copyright notice -
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+// CityHash, by Geoff Pike and Jyrki Alakuijala
+//
+// This file provides CityHash64() and related functions.
+//
+// It's probably possible to create even faster hash functions by
+// writing a program that systematically explores some of the space of
+// possible hash functions, by using SIMD instructions, or by
+// compromising on hash quality.
+
+#include <string.h>
+#include "city.h"
+
+static uint64 UNALIGNED_LOAD64(const char *p) {
+ uint64 result;
+ memcpy(&result, p, sizeof(result));
+ return result;
+}
+
+static uint32 UNALIGNED_LOAD32(const char *p) {
+ uint32 result;
+ memcpy(&result, p, sizeof(result));
+ return result;
+}
+
+#if !defined(WORDS_BIGENDIAN)
+
+#define uint32_in_expected_order(x) (x)
+#define uint64_in_expected_order(x) (x)
+
+#else
+
+#ifdef _MSC_VER
+#include <stdlib.h>
+#define bswap_32(x) _byteswap_ulong(x)
+#define bswap_64(x) _byteswap_uint64(x)
+
+#elif defined(__APPLE__)
+// Mac OS X / Darwin features
+#include <libkern/OSByteOrder.h>
+#define bswap_32(x) OSSwapInt32(x)
+#define bswap_64(x) OSSwapInt64(x)
+
+#else
+#include <byteswap.h>
+#endif
+
+#define uint32_in_expected_order(x) (bswap_32(x))
+#define uint64_in_expected_order(x) (bswap_64(x))
+
+#endif // WORDS_BIGENDIAN
+
+#if !defined(LIKELY)
+#if HAVE_BUILTIN_EXPECT
+#define LIKELY(x) (__builtin_expect(!!(x), 1))
+#else
+#define LIKELY(x) (x)
+#endif
+#endif
+
+static uint64 Fetch64(const char *p) {
+ return uint64_in_expected_order(UNALIGNED_LOAD64(p));
+}
+
+static uint32 Fetch32(const char *p) {
+ return uint32_in_expected_order(UNALIGNED_LOAD32(p));
+}
+
+// Some primes between 2^63 and 2^64 for various uses.
+static const uint64 k0 = 0xc3a5c85c97cb3127ULL;
+static const uint64 k1 = 0xb492b66fbe98f273ULL;
+static const uint64 k2 = 0x9ae16a3b2f90404fULL;
+static const uint64 k3 = 0xc949d7c7509e6557ULL;
+
+// Hash 128 input bits down to 64 bits of output.
+// This is intended to be a reasonably good hash function.
+inline uint64 Hash128to64(const uint128 x) {
+ // Murmur-inspired hashing.
+ const uint64 kMul = 0x9ddfea08eb382d69ULL;
+ uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul;
+ a ^= (a >> 47);
+ uint64 b = (Uint128High64(x) ^ a) * kMul;
+ b ^= (b >> 47);
+ b *= kMul;
+ return b;
+}
+
+
+// Bitwise right rotate. Normally this will compile to a single
+// instruction, especially if the shift is a manifest constant.
+static uint64 Rotate(uint64 val, int shift) {
+ // Avoid shifting by 64: doing so yields an undefined result.
+ return shift == 0 ? val : ((val >> shift) | (val << (64 - shift)));
+}
+
+// Equivalent to Rotate(), but requires the second arg to be non-zero.
+// On x86-64, and probably others, it's possible for this to compile
+// to a single instruction if both args are already in registers.
+static uint64 RotateByAtLeast1(uint64 val, int shift) {
+ return (val >> shift) | (val << (64 - shift));
+}
+
+static uint64 ShiftMix(uint64 val) {
+ return val ^ (val >> 47);
+}
+
+static uint64 HashLen16(uint64 u, uint64 v) {
+ uint128 result;
+ result.first = u;
+ result.second = v;
+ return Hash128to64(result);
+}
+
+static uint64 HashLen0to16(const char *s, size_t len) {
+ if (len > 8) {
+ uint64 a = Fetch64(s);
+ uint64 b = Fetch64(s + len - 8);
+ return HashLen16(a, RotateByAtLeast1(b + len, len)) ^ b;
+ }
+ if (len >= 4) {
+ uint64 a = Fetch32(s);
+ return HashLen16(len + (a << 3), Fetch32(s + len - 4));
+ }
+ if (len > 0) {
+ uint8 a = s[0];
+ uint8 b = s[len >> 1];
+ uint8 c = s[len - 1];
+ uint32 y = (uint32)(a) + ((uint32)(b) << 8);
+ uint32 z = len + ((uint32)(c) << 2);
+ return ShiftMix(y * k2 ^ z * k3) * k2;
+ }
+ return k2;
+}
+
+// This probably works well for 16-byte strings as well, but it may be overkill
+// in that case.
+static uint64 HashLen17to32(const char *s, size_t len) {
+ uint64 a = Fetch64(s) * k1;
+ uint64 b = Fetch64(s + 8);
+ uint64 c = Fetch64(s + len - 8) * k2;
+ uint64 d = Fetch64(s + len - 16) * k0;
+ return HashLen16(Rotate(a - b, 43) + Rotate(c, 30) + d,
+ a + Rotate(b ^ k3, 20) - c + len);
+}
+
+// Return a 16-byte hash for 48 bytes. Quick and dirty.
+// Callers do best to use "random-looking" values for a and b.
+// static pair<uint64, uint64> WeakHashLen32WithSeeds(
+uint128 WeakHashLen32WithSeeds6(
+ uint64 w, uint64 x, uint64 y, uint64 z, uint64 a, uint64 b) {
+ a += w;
+ b = Rotate(b + a + z, 21);
+ uint64 c = a;
+ a += x;
+ a += y;
+ b += Rotate(a, 44);
+
+ uint128 result;
+ result.first = (uint64) (a + z);
+ result.second = (uint64) (b + c);
+ return result;
+}
+
+// Return a 16-byte hash for s[0] ... s[31], a, and b. Quick and dirty.
+// static pair<uint64, uint64> WeakHashLen32WithSeeds(
+uint128 WeakHashLen32WithSeeds(
+ const char* s, uint64 a, uint64 b) {
+ return WeakHashLen32WithSeeds6(Fetch64(s),
+ Fetch64(s + 8),
+ Fetch64(s + 16),
+ Fetch64(s + 24),
+ a,
+ b);
+}
+
+// Return an 8-byte hash for 33 to 64 bytes.
+static uint64 HashLen33to64(const char *s, size_t len) {
+ uint64 z = Fetch64(s + 24);
+ uint64 a = Fetch64(s) + (len + Fetch64(s + len - 16)) * k0;
+ uint64 b = Rotate(a + z, 52);
+ uint64 c = Rotate(a, 37);
+ a += Fetch64(s + 8);
+ c += Rotate(a, 7);
+ a += Fetch64(s + 16);
+ uint64 vf = a + z;
+ uint64 vs = b + Rotate(a, 31) + c;
+ a = Fetch64(s + 16) + Fetch64(s + len - 32);
+ z = Fetch64(s + len - 8);
+ b = Rotate(a + z, 52);
+ c = Rotate(a, 37);
+ a += Fetch64(s + len - 24);
+ c += Rotate(a, 7);
+ a += Fetch64(s + len - 16);
+ uint64 wf = a + z;
+ uint64 ws = b + Rotate(a, 31) + c;
+ uint64 r = ShiftMix((vf + ws) * k2 + (wf + vs) * k0);
+ return ShiftMix(r * k0 + vs) * k2;
+}
+
+uint64 CityHash64(const char *s, size_t len) {
+ if (len <= 32) {
+ if (len <= 16) {
+ return HashLen0to16(s, len);
+ } else {
+ return HashLen17to32(s, len);
+ }
+ } else if (len <= 64) {
+ return HashLen33to64(s, len);
+ }
+
+ // For strings over 64 bytes we hash the end first, and then as we
+ // loop we keep 56 bytes of state: v, w, x, y, and z.
+ uint64 x = Fetch64(s + len - 40);
+ uint64 y = Fetch64(s + len - 16) + Fetch64(s + len - 56);
+ uint64 z = HashLen16(Fetch64(s + len - 48) + len, Fetch64(s + len - 24));
+ uint64 temp;
+ uint128 v = WeakHashLen32WithSeeds(s + len - 64, len, z);
+ uint128 w = WeakHashLen32WithSeeds(s + len - 32, y + k1, x);
+ x = x * k1 + Fetch64(s);
+
+ // Decrease len to the nearest multiple of 64, and operate on 64-byte chunks.
+ len = (len - 1) & ~(size_t)(63);
+ do {
+ x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+ y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
+ x ^= w.second;
+ y += v.first + Fetch64(s + 40);
+ z = Rotate(z + w.first, 33) * k1;
+ v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
+ w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
+ temp = z;
+ z = x;
+ x = temp;
+ s += 64;
+ len -= 64;
+ } while (len != 0);
+ return HashLen16(HashLen16(v.first, w.first) + ShiftMix(y) * k1 + z,
+ HashLen16(v.second, w.second) + x);
+}
+
+uint64 CityHash64WithSeed(const char *s, size_t len, uint64 seed) {
+ return CityHash64WithSeeds(s, len, k2, seed);
+}
+
+uint64 CityHash64WithSeeds(const char *s, size_t len,
+ uint64 seed0, uint64 seed1) {
+ return HashLen16(CityHash64(s, len) - seed0, seed1);
+}
+
+// A subroutine for CityHash128(). Returns a decent 128-bit hash for strings
+// of any length representable in signed long. Based on City and Murmur.
+static uint128 CityMurmur(const char *s, size_t len, uint128 seed) {
+ uint64 a = Uint128Low64(seed);
+ uint64 b = Uint128High64(seed);
+ uint64 c = 0;
+ uint64 d = 0;
+ signed long l = len - 16;
+ if (l <= 0) { // len <= 16
+ a = ShiftMix(a * k1) * k1;
+ c = b * k1 + HashLen0to16(s, len);
+ d = ShiftMix(a + (len >= 8 ? Fetch64(s) : c));
+ } else { // len > 16
+ c = HashLen16(Fetch64(s + len - 8) + k1, a);
+ d = HashLen16(b + len, c + Fetch64(s + len - 16));
+ a += d;
+ do {
+ a ^= ShiftMix(Fetch64(s) * k1) * k1;
+ a *= k1;
+ b ^= a;
+ c ^= ShiftMix(Fetch64(s + 8) * k1) * k1;
+ c *= k1;
+ d ^= c;
+ s += 16;
+ l -= 16;
+ } while (l > 0);
+ }
+ a = HashLen16(a, c);
+ b = HashLen16(d, b);
+
+ uint128 result;
+ result.first = (uint64) (a ^ b);
+ result.second = (uint64) (HashLen16(b,a));
+ return result;
+}
+
+uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed) {
+ if (len < 128) {
+ return CityMurmur(s, len, seed);
+ }
+
+ // We expect len >= 128 to be the common case. Keep 56 bytes of state:
+ // v, w, x, y, and z.
+ uint128 v, w;
+ uint64 x = Uint128Low64(seed);
+ uint64 y = Uint128High64(seed);
+ uint64 z = len * k1;
+ uint64 temp;
+ v.first = Rotate(y ^ k1, 49) * k1 + Fetch64(s);
+ v.second = Rotate(v.first, 42) * k1 + Fetch64(s + 8);
+ w.first = Rotate(y + z, 35) * k1 + x;
+ w.second = Rotate(x + Fetch64(s + 88), 53) * k1;
+
+ // This is the same inner loop as CityHash64(), manually unrolled.
+ do {
+ x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+ y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
+ x ^= w.second;
+ y += v.first + Fetch64(s + 40);
+ z = Rotate(z + w.first, 33) * k1;
+ v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
+ w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
+ temp = z;
+ z = x;
+ x = temp;
+ s += 64;
+ x = Rotate(x + y + v.first + Fetch64(s + 8), 37) * k1;
+ y = Rotate(y + v.second + Fetch64(s + 48), 42) * k1;
+ x ^= w.second;
+ y += v.first + Fetch64(s + 40);
+ z = Rotate(z + w.first, 33) * k1;
+ v = WeakHashLen32WithSeeds(s, v.second * k1, x + w.first);
+ w = WeakHashLen32WithSeeds(s + 32, z + w.second, y + Fetch64(s + 16));
+ temp = z;
+ z = x;
+ x = temp;
+ s += 64;
+ len -= 128;
+ } while (LIKELY(len >= 128));
+ x += Rotate(v.first + z, 49) * k0;
+ z += Rotate(w.first, 37) * k0;
+ // If 0 < len < 128, hash up to 4 chunks of 32 bytes each from the end of s.
+ size_t tail_done;
+ for (tail_done = 0; tail_done < len; ) {
+ tail_done += 32;
+ y = Rotate(x + y, 42) * k0 + v.second;
+ w.first += Fetch64(s + len - tail_done + 16);
+ x = x * k0 + w.first;
+ z += w.second + Fetch64(s + len - tail_done);
+ w.second += v.first;
+ v = WeakHashLen32WithSeeds(s + len - tail_done, v.first + z, v.second);
+ }
+ // At this point our 56 bytes of state should contain more than
+ // enough information for a strong 128-bit hash. We use two
+ // different 56-byte-to-8-byte hashes to get a 16-byte final result.
+ x = HashLen16(x, v.first);
+ y = HashLen16(y + z, w.first);
+
+ uint128 result;
+ result.first = (uint64) (HashLen16(x + v.second, w.second) + y);
+ result.second = (uint64) HashLen16(x + w.second, y + v.second);
+ return result;
+}
+
+uint128 CityHash128(const char *s, size_t len) {
+ uint128 r;
+ if (len >= 16) {
+ r.first = (uint64) (Fetch64(s) ^ k3);
+ r.second = (uint64) (Fetch64(s + 8));
+
+ return CityHash128WithSeed(s + 16,
+ len - 16,
+ r);
+
+ } else if (len >= 8) {
+ r.first = (uint64) (Fetch64(s) ^ (len * k0));
+ r.second = (uint64) (Fetch64(s + len - 8) ^ k1);
+
+ return CityHash128WithSeed(NULL,
+ 0,
+ r);
+ } else {
+ r.first = (uint64) k0;
+ r.second = (uint64) k1;
+ return CityHash128WithSeed(s, len, r);
+ }
+}
+
+#ifdef __SSE4_2__
+#include "citycrc.h"
+#include <nmmintrin.h>
+
+// Requires len >= 240.
+static void CityHashCrc256Long(const char *s, size_t len,
+ uint32 seed, uint64 *result) {
+ uint64 a = Fetch64(s + 56) + k0;
+ uint64 b = Fetch64(s + 96) + k0;
+ uint64 c = result[0] = HashLen16(b, len);
+ uint64 d = result[1] = Fetch64(s + 120) * k0 + len;
+ uint64 e = Fetch64(s + 184) + seed;
+ uint64 f = seed;
+ uint64 g = 0;
+ uint64 h = 0;
+ uint64 i = 0;
+ uint64 j = 0;
+ uint64 t = c + d;
+
+ // 240 bytes of input per iter.
+ size_t iters = len / 240;
+ len -= iters * 240;
+ do {
+#define CHUNK(multiplier, z) \
+ { \
+ uint64 old_a = a; \
+ a = Rotate(b, 41 ^ z) * multiplier + Fetch64(s); \
+ b = Rotate(c, 27 ^ z) * multiplier + Fetch64(s + 8); \
+ c = Rotate(d, 41 ^ z) * multiplier + Fetch64(s + 16); \
+ d = Rotate(e, 33 ^ z) * multiplier + Fetch64(s + 24); \
+ e = Rotate(t, 25 ^ z) * multiplier + Fetch64(s + 32); \
+ t = old_a; \
+ } \
+ f = _mm_crc32_u64(f, a); \
+ g = _mm_crc32_u64(g, b); \
+ h = _mm_crc32_u64(h, c); \
+ i = _mm_crc32_u64(i, d); \
+ j = _mm_crc32_u64(j, e); \
+ s += 40
+
+ CHUNK(1, 1); CHUNK(k0, 0);
+ CHUNK(1, 1); CHUNK(k0, 0);
+ CHUNK(1, 1); CHUNK(k0, 0);
+ } while (--iters > 0);
+
+ while (len >= 40) {
+ CHUNK(k0, 0);
+ len -= 40;
+ }
+ if (len > 0) {
+ s = s + len - 40;
+ CHUNK(k0, 0);
+ }
+ j += i << 32;
+ a = HashLen16(a, j);
+ h += g << 32;
+ b += h;
+ c = HashLen16(c, f) + i;
+ d = HashLen16(d, e + result[0]);
+ j += e;
+ i += HashLen16(h, t);
+ e = HashLen16(a, d) + j;
+ f = HashLen16(b, c) + a;
+ g = HashLen16(j, i) + c;
+ result[0] = e + f + g + h;
+ a = ShiftMix((a + g) * k0) * k0 + b;
+ result[1] += a + result[0];
+ a = ShiftMix(a * k0) * k0 + c;
+ result[2] = a + result[1];
+ a = ShiftMix((a + e) * k0) * k0;
+ result[3] = a + result[2];
+}
+
+// Requires len < 240.
+static void CityHashCrc256Short(const char *s, size_t len, uint64 *result) {
+ char buf[240];
+ memcpy(buf, s, len);
+ memset(buf + len, 0, 240 - len);
+ CityHashCrc256Long(buf, 240, ~(uint32)(len), result);
+}
+
+void CityHashCrc256(const char *s, size_t len, uint64 *result) {
+ if (LIKELY(len >= 240)) {
+ CityHashCrc256Long(s, len, 0, result);
+ } else {
+ CityHashCrc256Short(s, len, result);
+ }
+}
+
+uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed) {
+ if (len <= 900) {
+ return CityHash128WithSeed(s, len, seed);
+ } else {
+ uint64 result[4];
+ CityHashCrc256(s, len, result);
+ uint64 u = Uint128High64(seed) + result[0];
+ uint64 v = Uint128Low64(seed) + result[1];
+ uint128 crc;
+ crc.first = (uint64) (HashLen16(u, v + result[2]));
+ crc.second = (uint64) (HashLen16(Rotate(v, 32), u * k0 + result[3]));
+ return crc;
+ }
+}
+
+uint128 CityHashCrc128(const char *s, size_t len) {
+ if (len <= 900) {
+ return CityHash128(s, len);
+ } else {
+ uint64 result[4];
+ CityHashCrc256(s, len, result);
+ uint128 crc;
+ crc.first = (uint64) result[2];
+ crc.second = (uint64) result[3];
+ return crc;
+ }
+}
+
+#endif
View
86 src/city.h
@@ -0,0 +1,86 @@
+// city.h - cityhash-c
+// CityHash on C
+// Copyright (c) 2011-2012, Alexander Nusov
+//
+// - original copyright notice -
+// Copyright (c) 2011 Google, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+//
+// CityHash, by Geoff Pike and Jyrki Alakuijala
+//
+// This file provides a few functions for hashing strings. On x86-64
+// hardware in 2011, CityHash64() is faster than other high-quality
+// hash functions, such as Murmur. This is largely due to higher
+// instruction-level parallelism. CityHash64() and CityHash128() also perform
+// well on hash-quality tests.
+//
+// CityHash128() is optimized for relatively long strings and returns
+// a 128-bit hash. For strings more than about 2000 bytes it can be
+// faster than CityHash64().
+//
+// Functions in the CityHash family are not suitable for cryptography.
+//
+// WARNING: This code has not been tested on big-endian platforms!
+// It is known to work well on little-endian platforms that have a small penalty
+// for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs.
+//
+// By the way, for some hash functions, given strings a and b, the hash
+// of a+b is easily derived from the hashes of a and b. This property
+// doesn't hold for any hash functions in this file.
+
+#ifndef CITY_HASH_H_
+#define CITY_HASH_H_
+
+#include <stdlib.h>
+#include <stdint.h>
+
+typedef uint8_t uint8;
+typedef uint32_t uint32;
+typedef uint64_t uint64;
+
+typedef struct _uint128 uint128;
+struct _uint128 {
+ uint64 first;
+ uint64 second;
+};
+
+#define Uint128Low64(x) (x).first
+#define Uint128High64(x) (x).second
+
+// Hash function for a byte array.
+uint64 CityHash64(const char *buf, size_t len);
+
+// Hash function for a byte array. For convenience, a 64-bit seed is also
+// hashed into the result.
+uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed);
+
+// Hash function for a byte array. For convenience, two seeds are also
+// hashed into the result.
+uint64 CityHash64WithSeeds(const char *buf, size_t len,
+ uint64 seed0, uint64 seed1);
+
+// Hash function for a byte array.
+uint128 CityHash128(const char *s, size_t len);
+
+// Hash function for a byte array. For convenience, a 128-bit seed is also
+// hashed into the result.
+uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed);
+
+#endif // CITY_HASH_H_
View
77 src/dablooms.c
@@ -12,14 +12,14 @@
#include <sys/mman.h>
#include <unistd.h>
-#include "md5.h"
+#include "city.h"
#include "dablooms.h"
#define DABLOOMS_VERSION "0.8.1"
+#define DABLOOMS_HASH64(ptr, len, seed) CityHash64WithSeed(ptr, len, seed)
#define HEADER_BYTES (2*sizeof(uint32_t))
#define SCALE_HEADER_BYTES (3*sizeof(uint64_t))
-#define SALT_SIZE 16
const char *dablooms_version(void)
{
@@ -184,58 +184,41 @@ int bitmap_flush(bitmap_t *bitmap)
}
}
-/* Each function has a unique salt, so we need at least nfuncs salts.
- * An MD5 hash is 16 bytes long, and each salt only needds to be 4 bytes
- * Thus we can proportion 4 salts per each md5 hash we create as a salt.
- */
-void new_salts(counting_bloom_t *bloom)
+static void new_salts(counting_bloom_t *bloom)
{
- int div = bloom->nfuncs / 4;
- int mod = bloom->nfuncs % 4;
- int i;
+ const uint64_t root = 0xd32463a2ba01742cLL;
+ const uint64_t seed = 0x4fb95f06d3702acbLL;
+ int i, num_salts = bloom->nfuncs / 2;
- if (mod) {
- div += 1;
- }
- bloom->num_salts = div;
- bloom->salts = calloc(div, SALT_SIZE);
- for (i = 0; i < div; i++) {
- struct cvs_MD5Context context;
- unsigned char checksum[16];
- cvs_MD5Init (&context);
- cvs_MD5Update (&context, (unsigned char *) &i, sizeof(int));
- cvs_MD5Final (checksum, &context);
- memcpy(bloom->salts + i * SALT_SIZE, &checksum, SALT_SIZE);
+ if (bloom->nfuncs % 2)
+ num_salts++;
+
+ bloom->salts = calloc(num_salts, sizeof(uint64_t));
+ bloom->salts[0] = DABLOOMS_HASH64((char *)&root, sizeof(uint64_t), seed);
+
+ for (i = 1; i < num_salts; i++) {
+ uint64_t base = bloom->salts[i - 1] ^ i;
+ bloom->salts[i] = DABLOOMS_HASH64((char *)&base, sizeof(uint64_t), seed);
}
}
-/* We are are using the salts, adding them to the new md5 hash, adding the key,
- * converting said md5 hash to 4 byte indexes
- */
-unsigned int *hash_func(counting_bloom_t *bloom, const char *key, unsigned int *hashes)
+static void hash_func(counting_bloom_t *bloom, const char *key, uint32_t *hashes)
{
+ size_t key_len = strlen(key);
+ int i, num_salts = bloom->nfuncs / 2;
+ uint64_t *h = (uint64_t *)hashes;
- int i, j, hash_cnt, hash;
- unsigned char *salts = bloom->salts;
- hash_cnt = 0;
-
- for (i = 0; i < bloom->num_salts; i++) {
- struct cvs_MD5Context context;
- unsigned char checksum[16];
- cvs_MD5Init(&context);
- cvs_MD5Update(&context, salts + i * SALT_SIZE, SALT_SIZE);
- cvs_MD5Update(&context, (unsigned char *)key, strlen(key));
- cvs_MD5Final(checksum, &context);
- for (j = 0; j < sizeof(checksum); j += 4) {
- if (hash_cnt >= (bloom->nfuncs)) {
- break;
- }
- hash = *(uint32_t *)(checksum + j);
- hashes[hash_cnt] = hash % bloom->counts_per_func;
- hash_cnt++;
- }
- }
- return hashes;
+ for (i = 0; i < num_salts; ++i) {
+ h[i] = DABLOOMS_HASH64(key, key_len, bloom->salts[i]);
+ }
+
+ if (bloom->nfuncs % 2) {
+ uint32_t hash = (uint32_t)DABLOOMS_HASH64(key, key_len, bloom->salts[num_salts]);
+ hashes[bloom->nfuncs - 1] = hash;
+ }
+
+ for (i = 0; i < bloom->nfuncs; ++i)
+ hashes[i] = hashes[i] % bloom->counts_per_func;
}
int free_counting_bloom(counting_bloom_t *bloom)
View
7 src/dablooms.h
@@ -35,9 +35,10 @@ typedef struct {
unsigned int capacity;
unsigned int offset;
unsigned int counts_per_func;
- unsigned int num_salts;
- unsigned char *salts;
- unsigned int *hashes;
+
+ uint64_t *salts;
+ uint32_t *hashes;
+
size_t nfuncs;
size_t size;
size_t num_bytes;
View
332 src/md5.c
@@ -1,332 +0,0 @@
-/*
- * This code implements the MD5 message-digest algorithm.
- * The algorithm is due to Ron Rivest. This code was
- * written by Colin Plumb in 1993, no copyright is claimed.
- * This code is in the public domain; do with it what you wish.
- *
- * Equivalent code is available from RSA Data Security, Inc.
- * This code has been tested against that, and is equivalent,
- * except that you don't need to include two pages of legalese
- * with every copy.
- *
- * To compute the message digest of a chunk of bytes, declare an
- * MD5Context structure, pass it to MD5Init, call MD5Update as
- * needed on buffers full of bytes, and then call MD5Final, which
- * will fill a supplied 16-byte array with the digest.
- */
-
-/* This code was modified in 1997 by Jim Kingdon of Cyclic Software to
- not require an integer type which is exactly 32 bits. This work
- draws on the changes for the same purpose by Tatu Ylonen
- <ylo@cs.hut.fi> as part of SSH, but since I didn't actually use
- that code, there is no copyright issue. I hereby disclaim
- copyright in any changes I have made; this code remains in the
- public domain. */
-
-/* Note regarding cvs_* namespace: this avoids potential conflicts
- with libraries such as some versions of Kerberos. No particular
- need to worry about whether the system supplies an MD5 library, as
- this file is only about 3k of object code. */
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
-#include <string.h> /* for memcpy() and memset() */
-
-/* Add prototype support. */
-#ifndef PROTO
-#if defined (USE_PROTOTYPES) ? USE_PROTOTYPES : defined (__STDC__)
-#define PROTO(ARGS) ARGS
-#else
-#define PROTO(ARGS) ()
-#endif
-#endif
-
-#include "md5.h"
-
-/* Little-endian byte-swapping routines. Note that these do not
- depend on the size of datatypes such as cvs_uint32, nor do they require
- us to detect the endianness of the machine we are running on. It
- is possible they should be macros for speed, but I would be
- surprised if they were a performance bottleneck for MD5. */
-
-static cvs_uint32
-getu32 (addr)
- const unsigned char *addr;
-{
- return (((((unsigned long)addr[3] << 8) | addr[2]) << 8)
- | addr[1]) << 8 | addr[0];
-}
-
-static void
-putu32 (data, addr)
- cvs_uint32 data;
- unsigned char *addr;
-{
- addr[0] = (unsigned char)data;
- addr[1] = (unsigned char)(data >> 8);
- addr[2] = (unsigned char)(data >> 16);
- addr[3] = (unsigned char)(data >> 24);
-}
-
-/*
- * Start MD5 accumulation. Set bit count to 0 and buffer to mysterious
- * initialization constants.
- */
-void
-cvs_MD5Init (ctx)
- struct cvs_MD5Context *ctx;
-{
- ctx->buf[0] = 0x67452301;
- ctx->buf[1] = 0xefcdab89;
- ctx->buf[2] = 0x98badcfe;
- ctx->buf[3] = 0x10325476;
-
- ctx->bits[0] = 0;
- ctx->bits[1] = 0;
-}
-
-/*
- * Update context to reflect the concatenation of another buffer full
- * of bytes.
- */
-void
-cvs_MD5Update (ctx, buf, len)
- struct cvs_MD5Context *ctx;
- unsigned char const *buf;
- unsigned len;
-{
- cvs_uint32 t;
-
- /* Update bitcount */
-
- t = ctx->bits[0];
- if ((ctx->bits[0] = (t + ((cvs_uint32)len << 3)) & 0xffffffff) < t)
- ctx->bits[1]++; /* Carry from low to high */
- ctx->bits[1] += len >> 29;
-
- t = (t >> 3) & 0x3f; /* Bytes already in shsInfo->data */
-
- /* Handle any leading odd-sized chunks */
-
- if ( t ) {
- unsigned char *p = ctx->in + t;
-
- t = 64-t;
- if (len < t) {
- memcpy(p, buf, len);
- return;
- }
- memcpy(p, buf, t);
- cvs_MD5Transform (ctx->buf, ctx->in);
- buf += t;
- len -= t;
- }
-
- /* Process data in 64-byte chunks */
-
- while (len >= 64) {
- memcpy(ctx->in, buf, 64);
- cvs_MD5Transform (ctx->buf, ctx->in);
- buf += 64;
- len -= 64;
- }
-
- /* Handle any remaining bytes of data. */
-
- memcpy(ctx->in, buf, len);
-}
-
-/*
- * Final wrapup - pad to 64-byte boundary with the bit pattern
- * 1 0* (64-bit count of bits processed, MSB-first)
- */
-void
-cvs_MD5Final (digest, ctx)
- unsigned char digest[16];
- struct cvs_MD5Context *ctx;
-{
- unsigned count;
- unsigned char *p;
-
- /* Compute number of bytes mod 64 */
- count = (ctx->bits[0] >> 3) & 0x3F;
-
- /* Set the first char of padding to 0x80. This is safe since there is
- always at least one byte free */
- p = ctx->in + count;
- *p++ = 0x80;
-
- /* Bytes of padding needed to make 64 bytes */
- count = 64 - 1 - count;
-
- /* Pad out to 56 mod 64 */
- if (count < 8) {
- /* Two lots of padding: Pad the first block to 64 bytes */
- memset(p, 0, count);
- cvs_MD5Transform (ctx->buf, ctx->in);
-
- /* Now fill the next block with 56 bytes */
- memset(ctx->in, 0, 56);
- } else {
- /* Pad block to 56 bytes */
- memset(p, 0, count-8);
- }
-
- /* Append length in bits and transform */
- putu32(ctx->bits[0], ctx->in + 56);
- putu32(ctx->bits[1], ctx->in + 60);
-
- cvs_MD5Transform (ctx->buf, ctx->in);
- putu32(ctx->buf[0], digest);
- putu32(ctx->buf[1], digest + 4);
- putu32(ctx->buf[2], digest + 8);
- putu32(ctx->buf[3], digest + 12);
- memset(ctx, 0, sizeof(ctx)); /* In case it's sensitive */
-}
-
-#ifndef ASM_MD5
-
-/* The four core functions - F1 is optimized somewhat */
-
-/* #define F1(x, y, z) (x & y | ~x & z) */
-#define F1(x, y, z) (z ^ (x & (y ^ z)))
-#define F2(x, y, z) F1(z, x, y)
-#define F3(x, y, z) (x ^ y ^ z)
-#define F4(x, y, z) (y ^ (x | ~z))
-
-/* This is the central step in the MD5 algorithm. */
-#define MD5STEP(f, w, x, y, z, data, s) \
- ( w += f(x, y, z) + data, w &= 0xffffffff, w = w<<s | w>>(32-s), w += x )
-
-/*
- * The core of the MD5 algorithm, this alters an existing MD5 hash to
- * reflect the addition of 16 longwords of new data. MD5Update blocks
- * the data and converts bytes into longwords for this routine.
- */
-void
-cvs_MD5Transform (buf, inraw)
- cvs_uint32 buf[4];
- const unsigned char inraw[64];
-{
- register cvs_uint32 a, b, c, d;
- cvs_uint32 in[16];
- int i;
-
- for (i = 0; i < 16; ++i)
- in[i] = getu32 (inraw + 4 * i);
-
- a = buf[0];
- b = buf[1];
- c = buf[2];
- d = buf[3];
-
- MD5STEP(F1, a, b, c, d, in[ 0]+0xd76aa478, 7);
- MD5STEP(F1, d, a, b, c, in[ 1]+0xe8c7b756, 12);
- MD5STEP(F1, c, d, a, b, in[ 2]+0x242070db, 17);
- MD5STEP(F1, b, c, d, a, in[ 3]+0xc1bdceee, 22);
- MD5STEP(F1, a, b, c, d, in[ 4]+0xf57c0faf, 7);
- MD5STEP(F1, d, a, b, c, in[ 5]+0x4787c62a, 12);
- MD5STEP(F1, c, d, a, b, in[ 6]+0xa8304613, 17);
- MD5STEP(F1, b, c, d, a, in[ 7]+0xfd469501, 22);
- MD5STEP(F1, a, b, c, d, in[ 8]+0x698098d8, 7);
- MD5STEP(F1, d, a, b, c, in[ 9]+0x8b44f7af, 12);
- MD5STEP(F1, c, d, a, b, in[10]+0xffff5bb1, 17);
- MD5STEP(F1, b, c, d, a, in[11]+0x895cd7be, 22);
- MD5STEP(F1, a, b, c, d, in[12]+0x6b901122, 7);
- MD5STEP(F1, d, a, b, c, in[13]+0xfd987193, 12);
- MD5STEP(F1, c, d, a, b, in[14]+0xa679438e, 17);
- MD5STEP(F1, b, c, d, a, in[15]+0x49b40821, 22);
-
- MD5STEP(F2, a, b, c, d, in[ 1]+0xf61e2562, 5);
- MD5STEP(F2, d, a, b, c, in[ 6]+0xc040b340, 9);
- MD5STEP(F2, c, d, a, b, in[11]+0x265e5a51, 14);
- MD5STEP(F2, b, c, d, a, in[ 0]+0xe9b6c7aa, 20);
- MD5STEP(F2, a, b, c, d, in[ 5]+0xd62f105d, 5);
- MD5STEP(F2, d, a, b, c, in[10]+0x02441453, 9);
- MD5STEP(F2, c, d, a, b, in[15]+0xd8a1e681, 14);
- MD5STEP(F2, b, c, d, a, in[ 4]+0xe7d3fbc8, 20);
- MD5STEP(F2, a, b, c, d, in[ 9]+0x21e1cde6, 5);
- MD5STEP(F2, d, a, b, c, in[14]+0xc33707d6, 9);
- MD5STEP(F2, c, d, a, b, in[ 3]+0xf4d50d87, 14);
- MD5STEP(F2, b, c, d, a, in[ 8]+0x455a14ed, 20);
- MD5STEP(F2, a, b, c, d, in[13]+0xa9e3e905, 5);
- MD5STEP(F2, d, a, b, c, in[ 2]+0xfcefa3f8, 9);
- MD5STEP(F2, c, d, a, b, in[ 7]+0x676f02d9, 14);
- MD5STEP(F2, b, c, d, a, in[12]+0x8d2a4c8a, 20);
-
- MD5STEP(F3, a, b, c, d, in[ 5]+0xfffa3942, 4);
- MD5STEP(F3, d, a, b, c, in[ 8]+0x8771f681, 11);
- MD5STEP(F3, c, d, a, b, in[11]+0x6d9d6122, 16);
- MD5STEP(F3, b, c, d, a, in[14]+0xfde5380c, 23);
- MD5STEP(F3, a, b, c, d, in[ 1]+0xa4beea44, 4);
- MD5STEP(F3, d, a, b, c, in[ 4]+0x4bdecfa9, 11);
- MD5STEP(F3, c, d, a, b, in[ 7]+0xf6bb4b60, 16);
- MD5STEP(F3, b, c, d, a, in[10]+0xbebfbc70, 23);
- MD5STEP(F3, a, b, c, d, in[13]+0x289b7ec6, 4);
- MD5STEP(F3, d, a, b, c, in[ 0]+0xeaa127fa, 11);
- MD5STEP(F3, c, d, a, b, in[ 3]+0xd4ef3085, 16);
- MD5STEP(F3, b, c, d, a, in[ 6]+0x04881d05, 23);
- MD5STEP(F3, a, b, c, d, in[ 9]+0xd9d4d039, 4);
- MD5STEP(F3, d, a, b, c, in[12]+0xe6db99e5, 11);
- MD5STEP(F3, c, d, a, b, in[15]+0x1fa27cf8, 16);
- MD5STEP(F3, b, c, d, a, in[ 2]+0xc4ac5665, 23);
-
- MD5STEP(F4, a, b, c, d, in[ 0]+0xf4292244, 6);
- MD5STEP(F4, d, a, b, c, in[ 7]+0x432aff97, 10);
- MD5STEP(F4, c, d, a, b, in[14]+0xab9423a7, 15);
- MD5STEP(F4, b, c, d, a, in[ 5]+0xfc93a039, 21);
- MD5STEP(F4, a, b, c, d, in[12]+0x655b59c3, 6);
- MD5STEP(F4, d, a, b, c, in[ 3]+0x8f0ccc92, 10);
- MD5STEP(F4, c, d, a, b, in[10]+0xffeff47d, 15);
- MD5STEP(F4, b, c, d, a, in[ 1]+0x85845dd1, 21);
- MD5STEP(F4, a, b, c, d, in[ 8]+0x6fa87e4f, 6);
- MD5STEP(F4, d, a, b, c, in[15]+0xfe2ce6e0, 10);
- MD5STEP(F4, c, d, a, b, in[ 6]+0xa3014314, 15);
- MD5STEP(F4, b, c, d, a, in[13]+0x4e0811a1, 21);
- MD5STEP(F4, a, b, c, d, in[ 4]+0xf7537e82, 6);
- MD5STEP(F4, d, a, b, c, in[11]+0xbd3af235, 10);
- MD5STEP(F4, c, d, a, b, in[ 2]+0x2ad7d2bb, 15);
- MD5STEP(F4, b, c, d, a, in[ 9]+0xeb86d391, 21);
-
- buf[0] += a;
- buf[1] += b;
- buf[2] += c;
- buf[3] += d;
-}
-#endif
-
-#ifdef TEST
-/* Simple test program. Can use it to manually run the tests from
- RFC1321 for example. */
-#include <stdio.h>
-
-int
-main (int argc, char **argv)
-{
- struct cvs_MD5Context context;
- unsigned char checksum[16];
- int i;
- int j;
-
- if (argc < 2)
- {
- fprintf (stderr, "usage: %s string-to-hash\n", argv[0]);
- exit (1);
- }
- for (j = 1; j < argc; ++j)
- {
- printf ("MD5 (\"%s\") = ", argv[j]);
- cvs_MD5Init (&context);
- cvs_MD5Update (&context, argv[j], strlen (argv[j]));
- cvs_MD5Final (checksum, &context);
- for (i = 0; i < 16; i++)
- {
- printf ("%02x", (unsigned int) checksum[i]);
- }
- printf ("\n");
- }
- return 0;
-}
-#endif /* TEST */
View
39 src/md5.h
@@ -1,39 +0,0 @@
-/* See md5.c for explanation and copyright information. */
-
-/*
- * $FreeBSD: src/contrib/cvs/lib/md5.h,v 1.2 1999/12/11 15:10:02 peter Exp $
- */
-
-/* Add prototype support. */
-#ifndef PROTO
-#if defined (USE_PROTOTYPES) ? USE_PROTOTYPES : defined (__STDC__)
-#define PROTO(ARGS) ARGS
-#else
-#define PROTO(ARGS) ()
-#endif
-#endif
-
-#ifndef MD5_H
-#define MD5_H
-
-/* Unlike previous versions of this code, uint32 need not be exactly
- 32 bits, merely 32 bits or more. Choosing a data type which is 32
- bits instead of 64 is not important; speed is considerably more
- important. ANSI guarantees that "unsigned long" will be big enough,
- and always using it seems to have few disadvantages. */
-typedef unsigned long cvs_uint32;
-
-struct cvs_MD5Context {
- cvs_uint32 buf[4];
- cvs_uint32 bits[2];
- unsigned char in[64];
-};
-
-void cvs_MD5Init PROTO ((struct cvs_MD5Context *context));
-void cvs_MD5Update PROTO ((struct cvs_MD5Context *context,
- unsigned char const *buf, unsigned len));
-void cvs_MD5Final PROTO ((unsigned char digest[16],
- struct cvs_MD5Context *context));
-void cvs_MD5Transform PROTO ((cvs_uint32 buf[4], const unsigned char in[64]));
-
-#endif /* !MD5_H */

0 comments on commit 82a791d

Please sign in to comment.
Something went wrong with that request. Please try again.