diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd290cb --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*~ +.eunit +c_src/*.o +doc +ebin +priv diff --git a/README.md b/README.md new file mode 100644 index 0000000..d6a7c52 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +## Description + +The **erlang-murmurhash** library provides [MurmurHash](http://code.google.com/p/smhasher/) hash functions implemented in Erlang NIFs. + +## Building and Installing + +The **erlang-murmurhash** library is built with [rebar](https://github.com/basho/rebar), which must be in the command `PATH`. + +## Erlang Version + +The **erlang-murmurhash** library requires Erlang R14B or later. diff --git a/c_src/MurmurHash2.cpp b/c_src/MurmurHash2.cpp new file mode 100644 index 0000000..62fb1d4 --- /dev/null +++ b/c_src/MurmurHash2.cpp @@ -0,0 +1,276 @@ +#include "MurmurHash2.h" + +//----------------------------------------------------------------------------- +// MurmurHash2, by Austin Appleby + +// Note - This code makes a few assumptions about how your machine behaves - + +// 1. We can read a 4-byte value from any address without crashing +// 2. sizeof(int) == 4 + +// And it has a few limitations - + +// 1. It will not work incrementally. +// 2. It will not produce the same results on little-endian and big-endian +// machines. + +uint32_t MurmurHash2 ( const void * key, int len, uint32_t seed ) +{ + // 'm' and 'r' are mixing constants generated offline. + // They're not really 'magic', they just happen to work well. + + const uint32_t m = 0x5bd1e995; + const int r = 24; + + // Initialize the hash to a 'random' value + + uint32_t h = seed ^ len; + + // Mix 4 bytes at a time into the hash + + const unsigned char * data = (const unsigned char *)key; + + while(len >= 4) + { + uint32_t k = *(uint32_t*)data; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + // Handle the last few bytes of the input array + + switch(len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + // Do a few final mixes of the hash to ensure the last few + // bytes are well-incorporated. + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +//----------------------------------------------------------------------------- +// MurmurHash2, 64-bit versions, by Austin Appleby + +// The same caveats as 32-bit MurmurHash2 apply here - beware of alignment +// and endian-ness issues if used across multiple platforms. + +// 64-bit hash for 64-bit platforms + +uint64_t MurmurHash64A ( const void * key, int len, uint64_t seed ) +{ + const uint64_t m = 0xc6a4a7935bd1e995; + const int r = 47; + + uint64_t h = seed ^ (len * m); + + const uint64_t * data = (const uint64_t *)key; + const uint64_t * end = data + (len/8); + + while(data != end) + { + uint64_t k = *data++; + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + const unsigned char * data2 = (const unsigned char*)data; + + switch(len & 7) + { + case 7: h ^= uint64_t(data2[6]) << 48; + case 6: h ^= uint64_t(data2[5]) << 40; + case 5: h ^= uint64_t(data2[4]) << 32; + case 4: h ^= uint64_t(data2[3]) << 24; + case 3: h ^= uint64_t(data2[2]) << 16; + case 2: h ^= uint64_t(data2[1]) << 8; + case 1: h ^= uint64_t(data2[0]); + h *= m; + }; + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return h; +} + + +// 64-bit hash for 32-bit platforms + +uint64_t MurmurHash64B ( const void * key, int len, uint64_t seed ) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + uint32_t h1 = uint32_t(seed) ^ len; + uint32_t h2 = uint32_t(seed >> 32); + + const uint32_t * data = (const uint32_t *)key; + + while(len >= 8) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + + uint32_t k2 = *data++; + k2 *= m; k2 ^= k2 >> r; k2 *= m; + h2 *= m; h2 ^= k2; + len -= 4; + } + + if(len >= 4) + { + uint32_t k1 = *data++; + k1 *= m; k1 ^= k1 >> r; k1 *= m; + h1 *= m; h1 ^= k1; + len -= 4; + } + + switch(len) + { + case 3: h2 ^= ((unsigned char*)data)[2] << 16; + case 2: h2 ^= ((unsigned char*)data)[1] << 8; + case 1: h2 ^= ((unsigned char*)data)[0]; + h2 *= m; + }; + + h1 ^= h2 >> 18; h1 *= m; + h2 ^= h1 >> 22; h2 *= m; + h1 ^= h2 >> 17; h1 *= m; + h2 ^= h1 >> 19; h2 *= m; + + uint64_t h = h1; + + h = (h << 32) | h2; + + return h; +} + +//----------------------------------------------------------------------------- +// MurmurHash2A, by Austin Appleby + +// This is a variant of MurmurHash2 modified to use the Merkle-Damgard +// construction. Bulk speed should be identical to Murmur2, small-key speed +// will be 10%-20% slower due to the added overhead at the end of the hash. + +// This variant fixes a minor issue where null keys were more likely to +// collide with each other than expected, and also makes the function +// more amenable to incremental implementations. + +#define mmix(h,k) { k *= m; k ^= k >> r; k *= m; h *= m; h ^= k; } + +uint32_t MurmurHash2A ( const void * key, int len, uint32_t seed ) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + uint32_t l = len; + + const unsigned char * data = (const unsigned char *)key; + + uint32_t h = seed; + + while(len >= 4) + { + uint32_t k = *(uint32_t*)data; + + mmix(h,k); + + data += 4; + len -= 4; + } + + uint32_t t = 0; + + switch(len) + { + case 3: t ^= data[2] << 16; + case 2: t ^= data[1] << 8; + case 1: t ^= data[0]; + }; + + mmix(h,t); + mmix(h,l); + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +//----------------------------------------------------------------------------- +// MurmurHashNeutral2, by Austin Appleby + +// Same as MurmurHash2, but endian- and alignment-neutral. +// Half the speed though, alas. + +uint32_t MurmurHashNeutral2 ( const void * key, int len, uint32_t seed ) +{ + const uint32_t m = 0x5bd1e995; + const int r = 24; + + uint32_t h = seed ^ len; + + const unsigned char * data = (const unsigned char *)key; + + while(len >= 4) + { + uint32_t k; + + k = data[0]; + k |= data[1] << 8; + k |= data[2] << 16; + k |= data[3] << 24; + + k *= m; + k ^= k >> r; + k *= m; + + h *= m; + h ^= k; + + data += 4; + len -= 4; + } + + switch(len) + { + case 3: h ^= data[2] << 16; + case 2: h ^= data[1] << 8; + case 1: h ^= data[0]; + h *= m; + }; + + h ^= h >> 13; + h *= m; + h ^= h >> 15; + + return h; +} + +//----------------------------------------------------------------------------- + diff --git a/c_src/MurmurHash2.h b/c_src/MurmurHash2.h new file mode 100644 index 0000000..f0a4c76 --- /dev/null +++ b/c_src/MurmurHash2.h @@ -0,0 +1,7 @@ +#include + +uint32_t MurmurHash2 ( const void * key, int len, uint32_t seed ); +uint64_t MurmurHash64A ( const void * key, int len, uint64_t seed ); +uint64_t MurmurHash64B ( const void * key, int len, uint64_t seed ); +uint32_t MurmurHash2A ( const void * key, int len, uint32_t seed ); +uint32_t MurmurHashNeutral2 ( const void * key, int len, uint32_t seed ); diff --git a/c_src/erlang_murmurhash_nif.cpp b/c_src/erlang_murmurhash_nif.cpp new file mode 100644 index 0000000..8202f85 --- /dev/null +++ b/c_src/erlang_murmurhash_nif.cpp @@ -0,0 +1,245 @@ +#include +#include "MurmurHash2.h" + +extern "C" { + +ERL_NIF_TERM erlang_murmurhash2_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erlang_murmurhash2_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); + +ERL_NIF_TERM erlang_murmurhash64a_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erlang_murmurhash64a_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); + +ERL_NIF_TERM erlang_murmurhash64b_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erlang_murmurhash64b_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); + +ERL_NIF_TERM erlang_murmurhash2a_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erlang_murmurhash2a_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); + +ERL_NIF_TERM erlang_murmurhashneutral2_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); +ERL_NIF_TERM erlang_murmurhashneutral2_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]); + +} // extern "C" + +static ErlNifFunc nif_funcs[] = +{ + {"murmurhash2_impl", 1, erlang_murmurhash2_1_impl}, + {"murmurhash2_impl", 2, erlang_murmurhash2_2_impl}, + {"murmurhash64a_impl", 1, erlang_murmurhash64a_1_impl}, + {"murmurhash64a_impl", 2, erlang_murmurhash64a_2_impl}, + {"murmurhash64b_impl", 1, erlang_murmurhash64b_1_impl}, + {"murmurhash64b_impl", 2, erlang_murmurhash64b_2_impl}, + {"murmurhash2a_impl", 1, erlang_murmurhash2a_1_impl}, + {"murmurhash2a_impl", 2, erlang_murmurhash2a_2_impl}, + {"murmurhashneutral2_impl", 1, erlang_murmurhashneutral2_1_impl}, + {"murmurhashneutral2_impl", 2, erlang_murmurhashneutral2_2_impl} +}; + +static bool check_and_unpack_data(ErlNifEnv* env, ERL_NIF_TERM bin_term, + ErlNifBinary *bin); + +ERL_NIF_TERM +erlang_murmurhash2_1_impl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + h = MurmurHash2(bin.data, bin.size, 0); + + return enif_make_uint(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash2_2_impl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + uint32_t seed; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + if (!enif_get_uint(env, argv[1], &seed)) { + return enif_make_badarg(env); + } + + h = MurmurHash2(bin.data, bin.size, seed); + + return enif_make_uint(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash64a_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint64_t h; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + h = MurmurHash64A(bin.data, bin.size, 0); + + return enif_make_uint64(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash64a_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint64_t h; + uint64_t seed; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + if (!enif_get_uint64(env, argv[1], &seed)) { + return enif_make_badarg(env); + } + + h = MurmurHash64A(bin.data, bin.size, seed); + + return enif_make_uint64(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash64b_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint64_t h; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + h = MurmurHash64B(bin.data, bin.size, 0); + + return enif_make_uint64(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash64b_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint64_t h; + uint64_t seed; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + if (!enif_get_uint64(env, argv[1], &seed)) { + return enif_make_badarg(env); + } + + h = MurmurHash64B(bin.data, bin.size, seed); + + return enif_make_uint64(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash2a_1_impl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + h = MurmurHash2A(bin.data, bin.size, 0); + + return enif_make_uint(env, h); +} + +ERL_NIF_TERM +erlang_murmurhash2a_2_impl(ErlNifEnv* env, int argc, const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + uint32_t seed; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + if (!enif_get_uint(env, argv[1], &seed)) { + return enif_make_badarg(env); + } + + h = MurmurHash2A(bin.data, bin.size, seed); + + return enif_make_uint(env, h); +} + +ERL_NIF_TERM +erlang_murmurhashneutral2_1_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + h = MurmurHashNeutral2(bin.data, bin.size, 0); + + return enif_make_uint(env, h); +} + +ERL_NIF_TERM +erlang_murmurhashneutral2_2_impl(ErlNifEnv* env, int argc, + const ERL_NIF_TERM argv[]) +{ + ErlNifBinary bin; + uint32_t h; + uint32_t seed; + + if (!check_and_unpack_data(env, argv[0], &bin)) { + return enif_make_badarg(env); + } + + if (!enif_get_uint(env, argv[1], &seed)) { + return enif_make_badarg(env); + } + + h = MurmurHashNeutral2(bin.data, bin.size, seed); + + return enif_make_uint(env, h); +} + +bool +check_and_unpack_data(ErlNifEnv* env, ERL_NIF_TERM bin_term, ErlNifBinary *bin) +{ + if (!enif_is_binary(env, bin_term)) { + return false; + } + + if (!enif_inspect_binary(env, bin_term, bin)) { + return false; + } + + return true; +} + +ERL_NIF_INIT(erlang_murmurhash, nif_funcs, NULL, NULL, NULL, NULL); diff --git a/c_src/erlang_murmurhash_nif.h b/c_src/erlang_murmurhash_nif.h new file mode 100644 index 0000000..2b1a4dd --- /dev/null +++ b/c_src/erlang_murmurhash_nif.h @@ -0,0 +1,10 @@ +#ifndef __ERLANG_MURMURHASH_NIF_H_INCLUDED__ +#define __ERLANG_MURMURHASH_NIF_H_INCLUDED__ + +extern "C" { + +#include "MurmurHash2.h" + +} // extern "C" + +#endif diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..30061cb --- /dev/null +++ b/rebar.config @@ -0,0 +1,9 @@ +{require_otp_vsn, "R14"}. +{port_sources, ["c_src/*.cpp", "c_src/*.c"]}. +{so_name, "erlang_murmurhash_nif.so"}. + +{port_envs, [ + {"CXXFLAGS", "$CXXFLAGS -Wall -O3"}, + {"LDFLAGS", "$LDFLAGS -lstdc++"} + ]}. + diff --git a/src/erlang_murmurhash.app.src b/src/erlang_murmurhash.app.src new file mode 100644 index 0000000..34582ee --- /dev/null +++ b/src/erlang_murmurhash.app.src @@ -0,0 +1,14 @@ +{application, erlang_murmurhash, + [ + {description, "MurMurHash implemented in Erlang NIFs"}, + {vsn, "1.0"}, + {modules, [ + erlang_murmurhash + ]}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {env, []} + ]}. diff --git a/src/erlang_murmurhash.erl b/src/erlang_murmurhash.erl new file mode 100644 index 0000000..b5399e5 --- /dev/null +++ b/src/erlang_murmurhash.erl @@ -0,0 +1,143 @@ +%%% @author Konstantin Sorokin +%%% +%%% @copyright 2011 Konstantin V. Sorokin, All rights reserved. Open source, BSD License +%%% @version 1.0 +%%% +-module(erlang_murmurhash). +-version(1.0). +-on_load(init/0). +-export([murmurhash2/1, murmurhash2/2, + murmurhash2a/1, murmurhash2a/2, + murmurhashneutral2/1, murmurhashneutral2/2, + murmurhash64a/1, murmurhash64a/2, + murmurhash64b/1, murmurhash64b/2]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-spec init() -> ok | {error, any()}. + +%% @doc Initialize NIF. +init() -> + SoName = filename:join(case code:priv_dir(?MODULE) of + {error, bad_name} -> + %% this is here for testing purposes + filename:join( + [filename:dirname( + code:which(?MODULE)),"..","priv"]); + Dir -> + Dir + end, atom_to_list(?MODULE) ++ "_nif"), + erlang:load_nif(SoName, 0). + +murmurhash2_impl(_Data) -> + erlang:nif_error(not_loaded). + +murmurhash2_impl(_Data, _Seed) -> + erlang:nif_error(not_loaded). + +murmurhash2a_impl(_Data) -> + erlang:nif_error(not_loaded). + +murmurhash2a_impl(_Data, _Seed) -> + erlang:nif_error(not_loaded). + +murmurhash64a_impl(_Data) -> + erlang:nif_error(not_loaded). + +murmurhash64a_impl(_Data, _Seed) -> + erlang:nif_error(not_loaded). + +murmurhash64b_impl(_Data) -> + erlang:nif_error(not_loaded). + +murmurhash64b_impl(_Data, _Seed) -> + erlang:nif_error(not_loaded). + +murmurhashneutral2_impl(_Data) -> + erlang:nif_error(not_loaded). + +murmurhashneutral2_impl(_Data, _Seed) -> + erlang:nif_error(not_loaded). + +murmurhash2(Data) when is_binary(Data) -> + murmurhash2_impl(Data); +murmurhash2(Data) when is_list(Data) -> + murmurhash2_impl(list_to_binary(Data)); +murmurhash2(Data) when is_atom(Data) -> + murmurhash2_impl(term_to_binary(Data)). + +murmurhash2(Data, Seed) when is_binary(Data) andalso is_integer(Seed) -> + murmurhash2_impl(Data, Seed); +murmurhash2(Data, Seed) when is_list(Data) andalso is_integer(Seed) -> + murmurhash2_impl(list_to_binary(Data), Seed); +murmurhash2(Data, Seed) when is_atom(Data) andalso is_integer(Seed) -> + murmurhash2_impl(term_to_binary(Data), Seed). + +murmurhash2a(Data) when is_binary(Data) -> + murmurhash2a_impl(Data); +murmurhash2a(Data) when is_list(Data) -> + murmurhash2a_impl(list_to_binary(Data)); +murmurhash2a(Data) when is_atom(Data) -> + murmurhash2a_impl(term_to_binary(Data)). + +murmurhash2a(Data, Seed) when is_binary(Data) andalso is_integer(Seed) -> + murmurhash2a_impl(Data, Seed); +murmurhash2a(Data, Seed) when is_list(Data) andalso is_integer(Seed) -> + murmurhash2a_impl(list_to_binary(Data), Seed); +murmurhash2a(Data, Seed) when is_atom(Data) andalso is_integer(Seed) -> + murmurhash2a_impl(term_to_binary(Data), Seed). + +murmurhash64a(Data) when is_binary(Data) -> + murmurhash64a_impl(Data); +murmurhash64a(Data) when is_list(Data) -> + murmurhash64a_impl(list_to_binary(Data)); +murmurhash64a(Data) when is_atom(Data) -> + murmurhash64a_impl(term_to_binary(Data)). + +murmurhash64a(Data, Seed) when is_binary(Data) andalso is_integer(Seed) -> + murmurhash64a_impl(Data, Seed); +murmurhash64a(Data, Seed) when is_list(Data) andalso is_integer(Seed) -> + murmurhash64a_impl(list_to_binary(Data), Seed); +murmurhash64a(Data, Seed) when is_atom(Data) andalso is_integer(Seed) -> + murmurhash64a_impl(term_to_binary(Data), Seed). + +murmurhash64b(Data) when is_binary(Data) -> + murmurhash64b_impl(Data); +murmurhash64b(Data) when is_list(Data) -> + murmurhash64b_impl(list_to_binary(Data)); +murmurhash64b(Data) when is_atom(Data) -> + murmurhash64b_impl(term_to_binary(Data)). + +murmurhash64b(Data, Seed) when is_binary(Data) andalso is_integer(Seed) -> + murmurhash64b_impl(Data, Seed); +murmurhash64b(Data, Seed) when is_list(Data) andalso is_integer(Seed) -> + murmurhash64b_impl(list_to_binary(Data), Seed); +murmurhash64b(Data, Seed) when is_atom(Data) andalso is_integer(Seed) -> + murmurhash64b_impl(term_to_binary(Data), Seed). + +murmurhashneutral2(Data) when is_binary(Data) -> + murmurhashneutral2_impl(Data); +murmurhashneutral2(Data) when is_list(Data) -> + murmurhashneutral2_impl(list_to_binary(Data)); +murmurhashneutral2(Data) when is_atom(Data) -> + murmurhashneutral2_impl(term_to_binary(Data)). + +murmurhashneutral2(Data, Seed) when is_binary(Data) andalso is_integer(Seed) -> + murmurhashneutral2_impl(Data, Seed); +murmurhashneutral2(Data, Seed) when is_list(Data) andalso is_integer(Seed) -> + murmurhashneutral2_impl(list_to_binary(Data), Seed); +murmurhashneutral2(Data, Seed) when is_atom(Data) andalso is_integer(Seed) -> + murmurhashneutral2_impl(term_to_binary(Data), Seed). + +%% =================================================================== +%% EUnit tests +%% =================================================================== +-ifdef(TEST). + +%% To be written +erlang_murmurhash_test() -> + ok. + +-endif.