diff --git a/benchmarks/Benchmark.jl b/benchmarks/Benchmark.jl deleted file mode 100755 index dd20c77..0000000 --- a/benchmarks/Benchmark.jl +++ /dev/null @@ -1,100 +0,0 @@ -include("../src/Murmur3.jl") -using Murmur3 -using JSON -Murmur3.x86.hash32("Warmup") -Murmur3.x86.hash128("Warmup") -Murmur3.x64.hash128("Warmup") - -# Generate payload if it doesn't exist -if !isfile("benchmarks/data.txt") - println("Benchmark data doesn't exist, generating 256MiB..."); - f = open("benchmarks/data.txt", "w") - for i = 1:256*1024*1024 - write(f, (((i-1)*4)%25) + 'A'); - end - close(f) -end - -# Load the payloads -f = open("benchmarks/data.txt", "r") -payload_5B = read!(f, Array(Uint8,5)) -close(f) - -f = open("benchmarks/data.txt", "r") -payload_15B = read!(f, Array(Uint8,15)) -close(f) - -f = open("benchmarks/data.txt", "r") -payload_256B = read!(f, Array(Uint8,256)) -close(f) - -f = open("benchmarks/data.txt", "r") -payload_256KiB = read!(f, Array(Uint8,256 * 1024)) -close(f) - -f = open("benchmarks/data.txt", "r") -payload_256MiB = read!(f, Array(Uint8,256 * 1024 * 1024)) -close(f) - -# Load the C++ reference times -f = open("benchmarks/i5-3570K-4222-C-times.json") -reference = JSON.parse(readall(f)) -close(f) - - -function bench_x86_32(times::Uint32, payload::Array{Uint8}) - seed = uint32(0) - for i = 1:times - Murmur3.x86.hash32(payload, seed) - end -end - -function bench_x86_128(times::Uint32, payload::Array{Uint8}) - seed = uint32(0) - for i = 1:times - Murmur3.x86.hash128(payload, seed) - end -end - -function bench_x64_128(times::Uint32, payload::Array{Uint8}) - seed = uint64(0) - for i = 1:times - Murmur3.x64.hash128(payload, seed) - end -end - -function repeat_and_measure(benchfunc, ref, args...) - measurements = Float64[] - for i = 1:10 - tic() - apply(benchfunc, args) - push!(measurements, toq()) - end - m = mean(measurements ./ ref) - s = std(measurements ./ ref) - @printf("%.3f ± %.3f\n", m, s) -end - -function benchmark_scenarios(benchfunc, ref) - print(" 5 B: ") - repeat_and_measure(benchfunc, float(ref["pl_5B"]), uint32(100000000), payload_5B) - print(" 15 B: ") - repeat_and_measure(benchfunc, float(ref["pl_15B"]), uint32(50000000), payload_15B) - print("256 B: ") - repeat_and_measure(benchfunc, float(ref["pl_256B"]), uint32(5000000), payload_256B) - print("256 KiB: ") - repeat_and_measure(benchfunc, float(ref["pl_256KiB"]), uint32(5000), payload_256KiB) - print("256 MiB: ") - repeat_and_measure(benchfunc, float(ref["pl_256MiB"]), uint32(10), payload_256MiB) -end - -println("x86 32-bit") -benchmark_scenarios(bench_x86_32, reference["x86_32"]) - -println("x86 128-bit") -benchmark_scenarios(bench_x86_128, reference["x86_128"]) - -println("x64 128-bit") -benchmark_scenarios(bench_x64_128, reference["x64_128"]) - -println("Done!") diff --git a/benchmarks/C/Main.cpp b/benchmarks/C/Main.cpp deleted file mode 100644 index fc8e5c5..0000000 --- a/benchmarks/C/Main.cpp +++ /dev/null @@ -1,69 +0,0 @@ -#include "MurmurHash3.h" -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace std; - -string benchmark(char* payload, int size, unsigned int times, uint32_t* out, void (*fptr)(const void*, int, uint32_t, void*)) { - stringstream output; - output << "["; - for (unsigned int j = 0; j<10; j++) { - double start = omp_get_wtime(); - for (unsigned int i = 0; i - -#define ROTL32(x,y) _rotl(x,y) -#define ROTL64(x,y) _rotl64(x,y) - -#define BIG_CONSTANT(x) (x) - -// Other compilers - -#else // defined(_MSC_VER) - -#define FORCE_INLINE __attribute__((always_inline)) - -inline uint32_t rotl32 ( uint32_t x, int8_t r ) -{ - return (x << r) | (x >> (32 - r)); -} - -inline uint64_t rotl64 ( uint64_t x, int8_t r ) -{ - return (x << r) | (x >> (64 - r)); -} - -#define ROTL32(x,y) rotl32(x,y) -#define ROTL64(x,y) rotl64(x,y) - -#define BIG_CONSTANT(x) (x##LLU) - -#endif // !defined(_MSC_VER) - -//----------------------------------------------------------------------------- -// Block read - if your platform needs to do endian-swapping or can only -// handle aligned reads, do the conversion here - -FORCE_INLINE uint32_t getblock ( const uint32_t * p, int i ) -{ - return p[i]; -} - -FORCE_INLINE uint64_t getblock ( const uint64_t * p, int i ) -{ - return p[i]; -} - -//----------------------------------------------------------------------------- -// Finalization mix - force all bits of a hash block to avalanche - -FORCE_INLINE uint32_t fmix ( uint32_t h ) -{ - h ^= h >> 16; - h *= 0x85ebca6b; - h ^= h >> 13; - h *= 0xc2b2ae35; - h ^= h >> 16; - - return h; -} - -//---------- - -FORCE_INLINE uint64_t fmix ( uint64_t k ) -{ - k ^= k >> 33; - k *= BIG_CONSTANT(0xff51afd7ed558ccd); - k ^= k >> 33; - k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); - k ^= k >> 33; - - return k; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_32 ( const void * key, int len, - uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 4; - - uint32_t h1 = seed; - - uint32_t c1 = 0xcc9e2d51; - uint32_t c2 = 0x1b873593; - - //---------- - // body - - const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); - - for(int i = -nblocks; i; i++) - { - uint32_t k1 = getblock(blocks,i); - - k1 *= c1; - k1 = ROTL32(k1,15); - k1 *= c2; - - h1 ^= k1; - h1 = ROTL32(h1,13); - h1 = h1*5+0xe6546b64; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*4); - - uint32_t k1 = 0; - - switch(len & 3) - { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; - case 1: k1 ^= tail[0]; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; - - h1 = fmix(h1); - - *(uint32_t*)out = h1; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_128 ( const void * key, const int len, - uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 16; - - uint32_t h1 = seed; - uint32_t h2 = seed; - uint32_t h3 = seed; - uint32_t h4 = seed; - - uint32_t c1 = 0x239b961b; - uint32_t c2 = 0xab0e9789; - uint32_t c3 = 0x38b34ae5; - uint32_t c4 = 0xa1e38b93; - - //---------- - // body - - const uint32_t * blocks = (const uint32_t *)(data + nblocks*16); - - for(int i = -nblocks; i; i++) - { - uint32_t k1 = getblock(blocks,i*4+0); - uint32_t k2 = getblock(blocks,i*4+1); - uint32_t k3 = getblock(blocks,i*4+2); - uint32_t k4 = getblock(blocks,i*4+3); - - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - - h1 = ROTL32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b; - - k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; - - h2 = ROTL32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747; - - k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; - - h3 = ROTL32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35; - - k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; - - h4 = ROTL32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*16); - - uint32_t k1 = 0; - uint32_t k2 = 0; - uint32_t k3 = 0; - uint32_t k4 = 0; - - switch(len & 15) - { - case 15: k4 ^= tail[14] << 16; - case 14: k4 ^= tail[13] << 8; - case 13: k4 ^= tail[12] << 0; - k4 *= c4; k4 = ROTL32(k4,18); k4 *= c1; h4 ^= k4; - - case 12: k3 ^= tail[11] << 24; - case 11: k3 ^= tail[10] << 16; - case 10: k3 ^= tail[ 9] << 8; - case 9: k3 ^= tail[ 8] << 0; - k3 *= c3; k3 = ROTL32(k3,17); k3 *= c4; h3 ^= k3; - - case 8: k2 ^= tail[ 7] << 24; - case 7: k2 ^= tail[ 6] << 16; - case 6: k2 ^= tail[ 5] << 8; - case 5: k2 ^= tail[ 4] << 0; - k2 *= c2; k2 = ROTL32(k2,16); k2 *= c3; h2 ^= k2; - - case 4: k1 ^= tail[ 3] << 24; - case 3: k1 ^= tail[ 2] << 16; - case 2: k1 ^= tail[ 1] << 8; - case 1: k1 ^= tail[ 0] << 0; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; - - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; - - h1 = fmix(h1); - h2 = fmix(h2); - h3 = fmix(h3); - h4 = fmix(h4); - - h1 += h2; h1 += h3; h1 += h4; - h2 += h1; h3 += h1; h4 += h1; - - ((uint32_t*)out)[0] = h1; - ((uint32_t*)out)[1] = h2; - ((uint32_t*)out)[2] = h3; - ((uint32_t*)out)[3] = h4; -} - -//----------------------------------------------------------------------------- - -void MurmurHash3_x64_128 ( const void * key, const int len, - const uint32_t seed, void * out ) -{ - const uint8_t * data = (const uint8_t*)key; - const int nblocks = len / 16; - - uint64_t h1 = seed; - uint64_t h2 = seed; - - uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); - uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); - - //---------- - // body - - const uint64_t * blocks = (const uint64_t *)(data); - - for(int i = 0; i < nblocks; i++) - { - uint64_t k1 = getblock(blocks,i*2+0); - uint64_t k2 = getblock(blocks,i*2+1); - - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; - - h1 = ROTL64(h1,27); h1 += h2; h1 = h1*5+0x52dce729; - - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; - - h2 = ROTL64(h2,31); h2 += h1; h2 = h2*5+0x38495ab5; - } - - //---------- - // tail - - const uint8_t * tail = (const uint8_t*)(data + nblocks*16); - - uint64_t k1 = 0; - uint64_t k2 = 0; - - switch(len & 15) - { - case 15: k2 ^= uint64_t(tail[14]) << 48; - case 14: k2 ^= uint64_t(tail[13]) << 40; - case 13: k2 ^= uint64_t(tail[12]) << 32; - case 12: k2 ^= uint64_t(tail[11]) << 24; - case 11: k2 ^= uint64_t(tail[10]) << 16; - case 10: k2 ^= uint64_t(tail[ 9]) << 8; - case 9: k2 ^= uint64_t(tail[ 8]) << 0; - k2 *= c2; k2 = ROTL64(k2,33); k2 *= c1; h2 ^= k2; - - case 8: k1 ^= uint64_t(tail[ 7]) << 56; - case 7: k1 ^= uint64_t(tail[ 6]) << 48; - case 6: k1 ^= uint64_t(tail[ 5]) << 40; - case 5: k1 ^= uint64_t(tail[ 4]) << 32; - case 4: k1 ^= uint64_t(tail[ 3]) << 24; - case 3: k1 ^= uint64_t(tail[ 2]) << 16; - case 2: k1 ^= uint64_t(tail[ 1]) << 8; - case 1: k1 ^= uint64_t(tail[ 0]) << 0; - k1 *= c1; k1 = ROTL64(k1,31); k1 *= c2; h1 ^= k1; - }; - - //---------- - // finalization - - h1 ^= len; h2 ^= len; - - h1 += h2; - h2 += h1; - - h1 = fmix(h1); - h2 = fmix(h2); - - h1 += h2; - h2 += h1; - - ((uint64_t*)out)[0] = h1; - ((uint64_t*)out)[1] = h2; -} - -//----------------------------------------------------------------------------- - diff --git a/benchmarks/C/MurmurHash3.h b/benchmarks/C/MurmurHash3.h deleted file mode 100644 index 58e9820..0000000 --- a/benchmarks/C/MurmurHash3.h +++ /dev/null @@ -1,37 +0,0 @@ -//----------------------------------------------------------------------------- -// MurmurHash3 was written by Austin Appleby, and is placed in the public -// domain. The author hereby disclaims copyright to this source code. - -#ifndef _MURMURHASH3_H_ -#define _MURMURHASH3_H_ - -//----------------------------------------------------------------------------- -// Platform-specific functions and macros - -// Microsoft Visual Studio - -#if defined(_MSC_VER) - -typedef unsigned char uint8_t; -typedef unsigned long uint32_t; -typedef unsigned __int64 uint64_t; - -// Other compilers - -#else // defined(_MSC_VER) - -#include - -#endif // !defined(_MSC_VER) - -//----------------------------------------------------------------------------- - -void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); - -void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); - -void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); - -//----------------------------------------------------------------------------- - -#endif // _MURMURHASH3_H_ diff --git a/benchmarks/i5-3570K-4222-C-times.json b/benchmarks/i5-3570K-4222-C-times.json deleted file mode 100644 index 3c3e478..0000000 --- a/benchmarks/i5-3570K-4222-C-times.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "x86_32": { - "pl_5B": [0.481843, 0.481206, 0.481825, 0.481006, 0.481539, 0.481337, 0.481847, 0.480816, 0.481034, 0.481405], - "pl_15B": [0.35664, 0.356996, 0.356181, 0.356548, 0.356658, 0.356523, 0.356281, 0.35662, 0.355996, 0.356495], - "pl_256B": [0.386266, 0.386717, 0.386894, 0.386383, 0.385808, 0.386775, 0.386175, 0.388029, 0.390585, 0.388447], - "pl_256KiB": [0.390781, 0.389139, 0.386706, 0.386164, 0.386133, 0.386475, 0.38608, 0.386398, 0.387876, 0.385845], - "pl_256MiB": [0.796656, 0.795438, 0.797066, 0.797608, 0.798614, 0.804189, 0.797076, 0.799375, 0.797464, 0.795976] - }, - "x86_128": { - "pl_5B": [1.22192, 1.22188, 1.22216, 1.22207, 1.22324, 1.22262, 1.22291, 1.22292, 1.2233, 1.22235], - "pl_15B": [0.744801, 0.745251, 0.745304, 0.744549, 0.745118, 0.745368, 0.745109, 0.744712, 0.745404, 0.744987], - "pl_256B": [0.337077, 0.337135, 0.336579, 0.337161, 0.337057, 0.336539, 0.336663, 0.33687, 0.336891, 0.337103], - "pl_256KiB": [0.29451, 0.293992, 0.293979, 0.294191, 0.294423, 0.294539, 0.294441, 0.293962, 0.294119, 0.294112], - "pl_256MiB": [0.610099, 0.608672, 0.608947, 0.608948, 0.609375, 0.60935, 0.609084, 0.609287, 0.60862, 0.609891] - }, - "x64_128": { - "pl_5B": [0.799728, 0.79896, 0.798937, 0.799648, 0.798847, 0.79917, 0.79857, 0.800457, 0.799335, 0.798771], - "pl_15B": [0.528708, 0.528634, 0.528904, 0.529054, 0.528779, 0.529066, 0.528501, 0.528604, 0.528925, 0.528772], - "pl_256B": [0.186752, 0.186846, 0.186532, 0.187391, 0.187146, 0.186843, 0.186855, 0.187088, 0.186576, 0.186884], - "pl_256KiB": [0.165543, 0.165276, 0.16503, 0.16497, 0.164936, 0.165408, 0.165508, 0.165125, 0.164878, 0.165487], - "pl_256MiB": [0.347487, 0.345914, 0.346769, 0.346445, 0.346535, 0.347459, 0.346708, 0.347204, 0.345467, 0.346839] - } -} \ No newline at end of file diff --git a/src/Murmur3.jl b/src/Murmur3.jl index a3a7577..910a6d3 100644 --- a/src/Murmur3.jl +++ b/src/Murmur3.jl @@ -1,358 +1,6 @@ module Murmur3 - module x86 - - using Switch - - macro rotl32(var, value) - :(($var << $value) | ($var >> $(32-value))) - end - - macro fmix32(var) - quote - local value = $(var) - value $= (value >> 16) - value = uint32(value * uint32(0x85ebca6b)) - value $= (value >> 13) - value = uint32(value * uint32(0xc2b2ae35)) - value $= (value >> 16) - value - end - end - - function hash32(data::Array{Uint8}, seed::Uint32) - c1 = uint32(0xcc9e2d51) - c2 = uint32(0x1b873593) - h = seed - - len = uint32(length(data)) - remainder = len & 3 - blocks = div(len, 4) - p = convert(Ptr{Uint32}, pointer(data)) - - # Body - for next_block = 1:blocks - k = unsafe_load(p, next_block) - k = uint32(k * c1) - k = @rotl32(k, 15) - k = uint32(k * c2) - - h $= k - h = @rotl32(h, 13) - h = uint32(h * 5 + 0xe6546b64) - end - - # Tail - k = uint32(0) - last = blocks*4 + 1 - @switch remainder begin - @case 3 - k |= uint32(data[last + 2]) << 16 - @case 2 - k |= uint32(data[last + 1]) << 8 - @case 1 - k |= uint32(data[last]) - end - - k = uint32(k * c1) - k = @rotl32(k, 15) - k = uint32(k * c2) - h $= k - - # Finalization - h $= len - h = @fmix32(h) - - return h - - end - - hash32(data::Array{Uint8}, seed::Int) = hash32(data, uint32(seed)) - hash32(data::Array{Uint8}) = hash32(data, uint32(0)) - hash32(data::String, seed::Uint32) = hash32(convert(Array{Uint8}, data), seed) - hash32(data::String, seed::Int) = hash32(convert(Array{Uint8}, data), uint32(seed)) - hash32(data::String) = hash32(convert(Array{Uint8}, data), uint32(0)) - - function hash128(data::Array{Uint8}, seed::Uint32) - c1 = uint32(0x239b961b) - c2 = uint32(0xab0e9789) - c3 = uint32(0x38b34ae5) - c4 = uint32(0xa1e38b93) - - h1 = seed - h2 = seed - h3 = seed - h4 = seed - - len = uint32(length(data)) - remainder = len & 15 - blocks = div(len, 16) - p = convert(Ptr{Uint32}, pointer(data)) - - # Body - for next_block = 1:4:blocks*4 - k1 = unsafe_load(p, next_block) - k2 = unsafe_load(p, next_block+1) - k3 = unsafe_load(p, next_block+2) - k4 = unsafe_load(p, next_block+3) - - k1 = uint32(k1 * c1) - k1 = @rotl32(k1, 15) - k1 = uint32(k1 * c2) - h1 $= k1 - - h1 = @rotl32(h1, 19) - h1 = uint32(h1 + h2) - h1 = uint32(h1 * 5 + 0x561ccd1b) - - k2 = uint32(k2 * c2) - k2 = @rotl32(k2, 16) - k2 = uint32(k2 * c3) - h2 $= k2 - - h2 = @rotl32(h2, 17) - h2 = uint32(h2 + h3) - h2 = uint32(h2 * 5 + 0x0bcaa747) - - k3 = uint32(k3 * c3) - k3 = @rotl32(k3, 17) - k3 = uint32(k3 * c4) - h3 $= k3 - - h3 = @rotl32(h3, 15) - h3 = uint32(h3 + h4) - h3 = uint32(h3 * 5 + 0x96cd1c35) - - k4 = uint32(k4 * c4) - k4 = @rotl32(k4, 18) - k4 = uint32(k4 * c1) - h4 $= k4 - - h4 = @rotl32(h4, 13) - h4 = uint32(h4 + h1) - h4 = uint32(h4 * 5 + 0x32ac3b17) - - end - - # Tail - last = blocks*16 - k1 = uint32(0) - k2 = uint32(0) - k3 = uint32(0) - k4 = uint32(0) - - @switch remainder begin - @case 15 - k4 $= uint32(data[last+15]) << 16 - @case 14 - k4 $= uint32(data[last+14]) << 8 - @case 13 - k4 $= uint32(data[last+13]) - k4 = uint32(k4 * c4) - k4 = @rotl32(k4, 18) - k4 = uint32(k4 * c1) - h4 $= k4 - @case 12 - k3 $= uint32(data[last+12]) << 24 - @case 11 - k3 $= uint32(data[last+11]) << 16 - @case 10 - k3 $= uint32(data[last+10]) << 8 - @case 9 - k3 $= uint32(data[last+9]) - k3 = uint32(k3 * c3) - k3 = @rotl32(k3, 17) - k3 = uint32(k3 * c4) - h3 $= k3 - @case 8 - k2 $= uint32(data[last+8]) << 24 - @case 7 - k2 $= uint32(data[last+7]) << 16 - @case 6 - k2 $= uint32(data[last+6]) << 8 - @case 5 - k2 $= uint32(data[last+5]) - k2 = uint32(k2 * c2) - k2 = @rotl32(k2, 16) - k2 = uint32(k2 * c3) - h2 $= k2 - @case 4 - k1 $= uint32(data[last+4]) << 24 - @case 3 - k1 $= uint32(data[last+3]) << 16 - @case 2 - k1 $= uint32(data[last+2]) << 8 - @case 1 - k1 $= uint32(data[last+1]) - k1 = uint32(k1 * c1) - k1 = @rotl32(k1, 15) - k1 = uint32(k1 * c2) - h1 $= k1 - end - - # Finalization - h1 $= len - h2 $= len - h3 $= len - h4 $= len - - h1 = uint32(h1 + h2) - h1 = uint32(h1 + h3) - h1 = uint32(h1 + h4) - h2 = uint32(h2 + h1) - h3 = uint32(h3 + h1) - h4 = uint32(h4 + h1) - - h1 = @fmix32(h1) - h2 = @fmix32(h2) - h3 = @fmix32(h3) - h4 = @fmix32(h4) - - h1 = uint32(h1 + h2) - h1 = uint32(h1 + h3) - h1 = uint32(h1 + h4) - h2 = uint32(h2 + h1) - h3 = uint32(h3 + h1) - h4 = uint32(h4 + h1) - - return uint128(h1)<<96 | (uint128(h2)<<64) | (uint128(h3)<<32) | (uint128(h4)) - end - - hash128(data::Array{Uint8}, seed::Int) = hash128(data, uint32(seed)) - hash128(data::Array{Uint8}) = hash128(data, uint32(0)) - hash128(data::String, seed::Uint32) = hash128(convert(Array{Uint8}, data), seed) - hash128(data::String, seed::Int) = hash128(convert(Array{Uint8}, data), uint32(seed)) - hash128(data::String) = hash128(convert(Array{Uint8}, data), uint32(0)) - - end - - module x64 - - using Switch - - macro rotl64(var, value) - :(($var << $value) | ($var >> $(64-value))) - end - - macro fmix64(var) - quote - local value = $(var) - value $= (value >> 33) - value = uint64(value * uint64(0xff51afd7ed558ccd)) - value $= (value >> 33) - value = uint64(value * uint64(0xc4ceb9fe1a85ec53)) - value $= (value >> 33) - value - end - end - - function hash128(data::Array{Uint8}, seed::Uint64) - - c1 = uint64(0x87c37b91114253d5) - c2 = uint64(0x4cf5ad432745937f) - - h1 = seed - h2 = seed - - len = uint64(length(data)) - remainder = len & 15 - blocks = div(len, uint64(16)) - p = convert(Ptr{Uint64}, pointer(data)) - - # Body - iterations = uint64(blocks*2) - for next_block::Uint64 = uint64(1):uint64(2):iterations - k1 = unsafe_load(p, next_block) - k2 = unsafe_load(p, next_block+1) - - k1 = uint64(k1 * c1) - k1 = @rotl64(k1, 31) - k1 = uint64(k1 * c2) - h1 $= k1 - - h1 = @rotl64(h1, 27) - h1 = uint64(h1 + h2) - h1 = uint64(h1 * 5 + 0x52dce729) - - k2 = uint64(k2 * c2) - k2 = @rotl64(k2, 33) - k2 = uint64(k2 * c1) - h2 $= k2 - - h2 = @rotl64(h2, 31) - h2 = uint64(h2 + h1) - h2 = uint64(h2 * 5 + 0x38495ab5) - end - - # Tail - last = blocks*16 - k1 = uint64(0) - k2 = uint64(0) - - @switch remainder begin - @case 15 - k2 $= uint64(data[last+15]) << 48 - @case 14 - k2 $= uint64(data[last+14]) << 40 - @case 13 - k2 $= uint64(data[last+13]) << 32 - @case 12 - k2 $= uint64(data[last+12]) << 24 - @case 11 - k2 $= uint64(data[last+11]) << 16 - @case 10 - k2 $= uint64(data[last+10]) << 8 - @case 9 - k2 $= uint64(data[last+9]) << 0 - k2 = uint64(k2 * c2) - k2 = @rotl64(k2, 33) - k2 = uint64(k2 * c1) - h2 $= k2 - @case 8 - k1 $= uint64(data[last+8]) << 56 - @case 7 - k1 $= uint64(data[last+7]) << 48 - @case 6 - k1 $= uint64(data[last+6]) << 40 - @case 5 - k1 $= uint64(data[last+5]) << 32 - @case 4 - k1 $= uint64(data[last+4]) << 24 - @case 3 - k1 $= uint64(data[last+3]) << 16 - @case 2 - k1 $= uint64(data[last+2]) << 8 - @case 1 - k1 $= uint64(data[last+1]) - k1 = uint64(k1 * c1) - k1 = @rotl64(k1, 31) - k1 = uint64(k1 * c2) - h1 $= k1 - end - - # Finalization - h1 $= uint64(len) - h2 $= uint64(len) - h1 = uint64(h1 + h2) - h2 = uint64(h2 + h1) - h1 = @fmix64(h1) - h2 = @fmix64(h2) - h1 = uint64(h1 + h2) - h2 = uint64(h2 + h1) - - h1 = @rotl64(h1, 32) - h2 = @rotl64(h2, 32) - - return (uint128(h1) << 64) | uint128(h2) - - end - - hash128(data::Array{Uint8}, seed::Int) = hash128(data, uint64(seed)) - hash128(data::Array{Uint8}) = hash128(data, uint64(0)) - hash128(data::String, seed::Uint64) = hash128(convert(Array{Uint8}, data), seed) - hash128(data::String, seed::Int) = hash128(convert(Array{Uint8}, data), uint64(seed)) - hash128(data::String) = hash128(convert(Array{Uint8}, data), uint64(0)) - - end +include("x86.jl") +include("x64.jl") end diff --git a/src/x64.jl b/src/x64.jl new file mode 100644 index 0000000..877441a --- /dev/null +++ b/src/x64.jl @@ -0,0 +1,128 @@ +module x64 + +using Switch + +macro rotl64(var, value) + :(($var << $value) | ($var >> $(64-value))) +end + +macro fmix64(var) + quote + local value = $(var) + value $= (value >> 33) + value = UInt64(value * UInt64(0xff51afd7ed558ccd)) + value $= (value >> 33) + value = UInt64(value * UInt64(0xc4ceb9fe1a85ec53)) + value $= (value >> 33) + value + end +end + +function hash128(data::Array{UInt8}, seed::UInt64) + + c1 = UInt64(0x87c37b91114253d5) + c2 = UInt64(0x4cf5ad432745937f) + + h1 = seed + h2 = seed + + len = UInt64(length(data)) + remainder = len & 15 + blocks = div(len, UInt64(16)) + p = convert(Ptr{UInt64}, pointer(data)) + + # Body + iterations = UInt64(blocks*2) + for next_block::UInt64 = UInt64(1):UInt64(2):iterations + k1 = UInt64(unsafe_load(p, next_block)) + k2 = UInt64(unsafe_load(p, next_block+1)) + + k1 = k1 * c1 + k1 = @rotl64(k1, 31) + k1 = k1 * c2 + h1 $= k1 + + h1 = @rotl64(h1, 27) + h1 = h1 + h2 + h1 = h1 * UInt64(5) + UInt64(0x52dce729) + + k2 = k2 * c2 + k2 = @rotl64(k2, 33) + k2 = k2 * c1 + h2 $= k2 + + h2 = @rotl64(h2, 31) + h2 = h2 + h1 + h2 = h2 * UInt64(5) + UInt64(0x38495ab5) + end + + # Tail + last = blocks*16 + k1 = UInt64(0) + k2 = UInt64(0) + + @switch remainder begin + @case 15 + k2 $= UInt64(data[last+15]) << 48 + @case 14 + k2 $= UInt64(data[last+14]) << 40 + @case 13 + k2 $= UInt64(data[last+13]) << 32 + @case 12 + k2 $= UInt64(data[last+12]) << 24 + @case 11 + k2 $= UInt64(data[last+11]) << 16 + @case 10 + k2 $= UInt64(data[last+10]) << 8 + @case 9 + k2 $= UInt64(data[last+9]) << 0 + k2 = k2 * c2 + k2 = @rotl64(k2, 33) + k2 = k2 * c1 + h2 $= k2 + @case 8 + k1 $= UInt64(data[last+8]) << 56 + @case 7 + k1 $= UInt64(data[last+7]) << 48 + @case 6 + k1 $= UInt64(data[last+6]) << 40 + @case 5 + k1 $= UInt64(data[last+5]) << 32 + @case 4 + k1 $= UInt64(data[last+4]) << 24 + @case 3 + k1 $= UInt64(data[last+3]) << 16 + @case 2 + k1 $= UInt64(data[last+2]) << 8 + @case 1 + k1 $= UInt64(data[last+1]) + k1 = k1 * c1 + k1 = @rotl64(k1, 31) + k1 = k1 * c2 + h1 $= k1 + end + + # Finalization + h1 $= len + h2 $= len + h1 = h1 + h2 + h2 = h2 + h1 + h1 = @fmix64(h1) + h2 = @fmix64(h2) + h1 = h1 + h2 + h2 = h2 + h1 + + h1 = @rotl64(h1, 32) + h2 = @rotl64(h2, 32) + + return (UInt128(h1) << 64) | UInt128(h2) + +end + +hash128(data::Array{UInt8}, seed::Int) = hash128(data, UInt64(seed)) +hash128(data::Array{UInt8}) = hash128(data, UInt64(0)) +hash128(data::AbstractString, seed::UInt64) = hash128(convert(Array{UInt8}, data), seed) +hash128(data::AbstractString, seed::Int) = hash128(convert(Array{UInt8}, data), UInt64(seed)) +hash128(data::AbstractString) = hash128(convert(Array{UInt8}, data), UInt64(0)) + +end diff --git a/src/x86.jl b/src/x86.jl new file mode 100644 index 0000000..cfc9d82 --- /dev/null +++ b/src/x86.jl @@ -0,0 +1,225 @@ +module x86 + +using Switch + +macro rotl32(var, value) + :(($var << $value) | ($var >> $(32-value))) +end + +macro fmix32(var) + quote + local value = $(var) + value $= (value >> 16) + value = UInt32(value * UInt32(0x85ebca6b)) + value $= (value >> 13) + value = UInt32(value * UInt32(0xc2b2ae35)) + value $= (value >> 16) + value + end +end + +function hash32(data::Array{UInt8}, seed::UInt32) + c1 = UInt32(0xcc9e2d51) + c2 = UInt32(0x1b873593) + h = seed + + len = UInt32(length(data)) + remainder = len & 3 + blocks = div(len, 4) + p = convert(Ptr{UInt32}, pointer(data)) + + # Body + for next_block = 1:blocks + k = UInt32(unsafe_load(p, next_block)) + k = k * c1 + k = @rotl32(k, 15) + k = k * c2 + + h $= k + h = @rotl32(h, 13) + h = h * UInt32(5) + UInt32(0xe6546b64) + end + + # Tail + k = UInt32(0) + last = blocks*4 + 1 + @switch remainder begin + @case 3 + k |= UInt32(data[last + 2]) << 16 + @case 2 + k |= UInt32(data[last + 1]) << 8 + @case 1 + k |= UInt32(data[last]) + end + + k = k * c1 + k = @rotl32(k, 15) + k = k * c2 + h $= k + + # Finalization + h $= len + h = @fmix32(h) + + return h + +end + +hash32(data::Array{UInt8}, seed::Int) = hash32(data, UInt32(seed)) +hash32(data::Array{UInt8}) = hash32(data, UInt32(0)) +hash32(data::AbstractString, seed::UInt32) = hash32(convert(Array{UInt8}, data), seed) +hash32(data::AbstractString, seed::Int) = hash32(convert(Array{UInt8}, data), UInt32(seed)) +hash32(data::AbstractString) = hash32(convert(Array{UInt8}, data), UInt32(0)) + +function hash128(data::Array{UInt8}, seed::UInt32) + c1 = UInt32(0x239b961b) + c2 = UInt32(0xab0e9789) + c3 = UInt32(0x38b34ae5) + c4 = UInt32(0xa1e38b93) + + h1 = seed + h2 = seed + h3 = seed + h4 = seed + + len = UInt32(length(data)) + remainder = len & 15 + blocks = div(len, 16) + p = convert(Ptr{UInt32}, pointer(data)) + + # Body + for next_block = 1:4:blocks*4 + k1 = UInt32(unsafe_load(p, next_block)) + k2 = UInt32(unsafe_load(p, next_block+1)) + k3 = UInt32(unsafe_load(p, next_block+2)) + k4 = UInt32(unsafe_load(p, next_block+3)) + + k1 = k1 * c1 + k1 = @rotl32(k1, 15) + k1 = k1 * c2 + h1 $= k1 + + h1 = @rotl32(h1, 19) + h1 = h1 + h2 + h1 = h1 * UInt32(5) + UInt32(0x561ccd1b) + + k2 = k2 * c2 + k2 = @rotl32(k2, 16) + k2 = k2 * c3 + h2 $= k2 + + h2 = @rotl32(h2, 17) + h2 = h2 + h3 + h2 = h2 * UInt32(5) + UInt32(0x0bcaa747) + + k3 = k3 * c3 + k3 = @rotl32(k3, 17) + k3 = k3 * c4 + h3 $= k3 + + h3 = @rotl32(h3, 15) + h3 = h3 + h4 + h3 = h3 * UInt32(5) + UInt32(0x96cd1c35) + + k4 = k4 * c4 + k4 = @rotl32(k4, 18) + k4 = k4 * c1 + h4 $= k4 + + h4 = @rotl32(h4, 13) + h4 = h4 + h1 + h4 = h4 * UInt32(5) + 0x32ac3b17 + + end + + # Tail + last = blocks*16 + k1 = UInt32(0) + k2 = UInt32(0) + k3 = UInt32(0) + k4 = UInt32(0) + + @switch remainder begin + @case 15 + k4 $= UInt32(data[last+15]) << 16 + @case 14 + k4 $= UInt32(data[last+14]) << 8 + @case 13 + k4 $= UInt32(data[last+13]) + k4 = k4 * c4 + k4 = @rotl32(k4, 18) + k4 = k4 * c1 + h4 $= k4 + @case 12 + k3 $= UInt32(data[last+12]) << 24 + @case 11 + k3 $= UInt32(data[last+11]) << 16 + @case 10 + k3 $= UInt32(data[last+10]) << 8 + @case 9 + k3 $= UInt32(data[last+9]) + k3 = k3 * c3 + k3 = @rotl32(k3, 17) + k3 = k3 * c4 + h3 $= k3 + @case 8 + k2 $= UInt32(data[last+8]) << 24 + @case 7 + k2 $= UInt32(data[last+7]) << 16 + @case 6 + k2 $= UInt32(data[last+6]) << 8 + @case 5 + k2 $= UInt32(data[last+5]) + k2 = k2 * c2 + k2 = @rotl32(k2, 16) + k2 = k2 * c3 + h2 $= k2 + @case 4 + k1 $= UInt32(data[last+4]) << 24 + @case 3 + k1 $= UInt32(data[last+3]) << 16 + @case 2 + k1 $= UInt32(data[last+2]) << 8 + @case 1 + k1 $= UInt32(data[last+1]) + k1 = k1 * c1 + k1 = @rotl32(k1, 15) + k1 = k1 * c2 + h1 $= k1 + end + + # Finalization + h1 $= len + h2 $= len + h3 $= len + h4 $= len + + h1 = h1 + h2 + h1 = h1 + h3 + h1 = h1 + h4 + h2 = h2 + h1 + h3 = h3 + h1 + h4 = h4 + h1 + + h1 = @fmix32(h1) + h2 = @fmix32(h2) + h3 = @fmix32(h3) + h4 = @fmix32(h4) + + h1 = h1 + h2 + h1 = h1 + h3 + h1 = h1 + h4 + h2 = h2 + h1 + h3 = h3 + h1 + h4 = h4 + h1 + + return UInt128(h1)<<96 | (UInt128(h2)<<64) | (UInt128(h3)<<32) | (UInt128(h4)) +end + +hash128(data::Array{UInt8}, seed::Int) = hash128(data, UInt32(seed)) +hash128(data::Array{UInt8}) = hash128(data, UInt32(0)) +hash128(data::AbstractString, seed::UInt32) = hash128(convert(Array{UInt8}, data), seed) +hash128(data::AbstractString, seed::Int) = hash128(convert(Array{UInt8}, data), UInt32(seed)) +hash128(data::AbstractString) = hash128(convert(Array{UInt8}, data), UInt32(0)) + +end diff --git a/test/runtests.jl b/test/runtests.jl index 1b969b3..13b5292 100755 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -26,20 +26,20 @@ for test in tests end # Test multiple dispatch -@test Murmur3.x86.hash32(Uint8[1, 2, 3], 5) == uint32(0x4c298f63) -@test Murmur3.x86.hash32(Uint8[1, 2, 3]) == uint32(0x80d1d204) -@test Murmur3.x86.hash32("Test", uint32(5)) == uint32(0x578de544) -@test Murmur3.x86.hash32("Test", 5) == uint32(0x578de544) -@test Murmur3.x86.hash32("Test") == uint32(0x07556ca6) - -@test Murmur3.x86.hash128(Uint8[1, 2, 3], 5) == uint128(0x5c845b3cf61b38eef61b38eef61b38ee) -@test Murmur3.x86.hash128(Uint8[1, 2, 3]) == uint128(0xf60164e1b5134233b5134233b5134233) -@test Murmur3.x86.hash128("Test", uint32(5)) == uint128(0xfca9493ccf980d40cf980d40cf980d40) -@test Murmur3.x86.hash128("Test", 5) == uint128(0xfca9493ccf980d40cf980d40cf980d40) -@test Murmur3.x86.hash128("Test") == uint128(0x1bccc51416e53d6c16e53d6c16e53d6c) - -@test Murmur3.x64.hash128(Uint8[1, 2, 3], 5) == uint128(0x88451581d843d60150f5d4a75a40b6c2) -@test Murmur3.x64.hash128(Uint8[1, 2, 3]) == uint128(0x0e1337a91a643eef3c239a65494e4a40) -@test Murmur3.x64.hash128("Test", uint64(5)) == uint128(0x536b3b8cfcdf124c3ea8439527849576) -@test Murmur3.x64.hash128("Test", 5) == uint128(0x536b3b8cfcdf124c3ea8439527849576) -@test Murmur3.x64.hash128("Test") == uint128(0x9eac3743ee31bc6fbf2185a737a1b11a) +@test Murmur3.x86.hash32(UInt8[1, 2, 3], 5) == UInt32(0x4c298f63) +@test Murmur3.x86.hash32(UInt8[1, 2, 3]) == UInt32(0x80d1d204) +@test Murmur3.x86.hash32("Test", UInt32(5)) == UInt32(0x578de544) +@test Murmur3.x86.hash32("Test", 5) == UInt32(0x578de544) +@test Murmur3.x86.hash32("Test") == UInt32(0x07556ca6) + +@test Murmur3.x86.hash128(UInt8[1, 2, 3], 5) == UInt128(0x5c845b3cf61b38eef61b38eef61b38ee) +@test Murmur3.x86.hash128(UInt8[1, 2, 3]) == UInt128(0xf60164e1b5134233b5134233b5134233) +@test Murmur3.x86.hash128("Test", UInt32(5)) == UInt128(0xfca9493ccf980d40cf980d40cf980d40) +@test Murmur3.x86.hash128("Test", 5) == UInt128(0xfca9493ccf980d40cf980d40cf980d40) +@test Murmur3.x86.hash128("Test") == UInt128(0x1bccc51416e53d6c16e53d6c16e53d6c) + +@test Murmur3.x64.hash128(UInt8[1, 2, 3], 5) == UInt128(0x88451581d843d60150f5d4a75a40b6c2) +@test Murmur3.x64.hash128(UInt8[1, 2, 3]) == UInt128(0x0e1337a91a643eef3c239a65494e4a40) +@test Murmur3.x64.hash128("Test", UInt64(5)) == UInt128(0x536b3b8cfcdf124c3ea8439527849576) +@test Murmur3.x64.hash128("Test", 5) == UInt128(0x536b3b8cfcdf124c3ea8439527849576) +@test Murmur3.x64.hash128("Test") == UInt128(0x9eac3743ee31bc6fbf2185a737a1b11a)