c/fnv.c c/fnv.h c/fnv_drv.c c/murmur.c c/murmur.h c/murmur_drv.c elibs/fnv.erl elibs/murmur.erl @@ -4,4 +4,6 @@ Authors of dynomite. Todd Lipcon <todd@lipcon.org> Jason Pellerin <jpellerin@gmail.com> -The plumbing for the couch storage engine is from the couchdb project: http://incubator.apache.org/couchdb/ \ No newline at end of file +The plumbing for the couch storage engine is from the couchdb project: http://incubator.apache.org/couchdb/ + +MurmurHash2 is courtesy of Austin Appleby \ No newline at end of file AUTHORS @@ -2,10 +2,15 @@ require 'fileutils' require 'rubygems' require 'rake' +require 'rake/clean' ERLC_TEST_FLAGS = "-pa deps/eunit/ebin -I deps/eunit/include -DTEST" ERLC_FLAGS = "+debug_info -W0 -I include -pa deps/mochiweb/ebin -I deps/mochiweb/include -pa deps/rfc4627/ebin -I deps/rfc4627/include -I gen-erl/ -o ebin" +CLEAN.include("ebin/*.beam") +CLEAN.include("c/*.o") +CLEAN.include("lib/*.so") + task :default => [:build_deps, :build_c_drivers] do puts "building #{ENV['TEST']}" sh "erlc #{ERLC_FLAGS} #{ENV['TEST'] ? ERLC_TEST_FLAGS : ''} elibs/*.erl gen-erl/*.erl" @@ -18,15 +23,6 @@ task :default => [:build_deps, :build_c_drivers] do # end end -task :clean_test do - sh "rm -rf etest/log/*" -end - -task :clean => [:clean_test] do - sh "rm -f ebin/*.beam" - sh "rm -f etest/*.beam" -end - task :test_env => [:build_test_deps, :test_config] do puts "test env" ENV['TEST'] = 'test' @@ -115,7 +111,7 @@ task :c_env do ERLDIR = `awk -F= '/ROOTDIR=/ { print $2; exit; }' #{ERL}`.chomp ERTSBASE = `erl -noshell -noinput -eval 'io:format (\"~s\", [[ \"/\" ++ filename:join (lists:reverse ([ \"erts-\" ++ erlang:system_info (version) | tl (lists:reverse (string:tokens (code:lib_dir (), \"/\"))) ])) ]]).' -s erlang halt `.chomp ERL_INTERFACE = `ls #{ERLDIR}/lib`.split("\n").grep(/erl_interface/).last - CPPFLAGS = "-I #{ERTSBASE}/include -I #{ERLDIR}/lib/#{ERL_INTERFACE}/include -Wall -fPIC -I./" + CPPFLAGS = "-I #{ERTSBASE}/include -I #{ERLDIR}/lib/#{ERL_INTERFACE}/include -Wall -g -O2 -fPIC -I./" LIBEI = "#{ERLDIR}/lib/#{ERL_INTERFACE}/lib/libei.a" if `uname` =~ /Linux/ LDFLAGS = " -shared" @@ -164,9 +160,11 @@ DRIVERS = FileList['c/*_drv.c'].pathmap("%{c,lib}X.so") directory "lib" -rule ".so" => '%{lib,c}X.o' do |t| - puts "cc #{CPPFLAGS} #{LDFLAGS} -o #{t.name} #{t.source} #{LIBEI}" - sh "cc #{CPPFLAGS} #{LDFLAGS} -o #{t.name} #{t.source} #{LIBEI}" +# task "lib/murmur_drv.c" => ["c/murmur.o"] + +rule ".so" => ['%{lib,c}X.o', 'c/murmur.o', 'c/fnv.o'] do |t| + puts "cc #{CPPFLAGS} #{LDFLAGS} -o #{t.name} #{t.prerequisites.join(' ')} #{LIBEI}" + sh "cc #{CPPFLAGS} #{LDFLAGS} -o #{t.name} #{t.prerequisites.join(' ')} #{LIBEI}" end rule ".o" => ".c" do |t| @@ -174,4 +172,6 @@ rule ".o" => ".c" do |t| sh "cc #{CPPFLAGS} -c -o #{t.name} #{t.source}" end + + task :build_c_drivers => [:c_env, "lib"] + DRIVERS \ No newline at end of file Rakefile @@ -8,15 +8,17 @@ start() -> application:load(dynomite), % spawn(fun() -> collect_loop() end), crypto:start(), + murmur:start(), + fnv:start(), application:start(dynomite). collect_loop() -> process_flag(trap_exit, true), - Filename = io_lib:format("/home/cliff/~w-dyn.dump", [lib_misc:now_int()]), + Filename = io_lib:format("/home/cliff/dumps/~w-dyn.dump", [lib_misc:now_int()]), sys_info(Filename), receive nothing -> ok - after 15000 -> collect_loop() + after 5000 -> collect_loop() end. sys_info(Filename) -> elibs/dynomite.erl @@ -4,7 +4,7 @@ -define(OFFSET_BASIS, 2166136261). -define(FNV_PRIME, 16777619). --export([pmap/3, hash/1, nthdelete/2, zero_split/1, nthreplace/3, rand_str/1, position/2, shuffle/1, floor/1, ceiling/1, time_to_epoch_int/1, time_to_epoch_float/1, now_int/0, now_float/0, byte_size/1, listify/1, reverse_bits/1]). +-export([pmap/3, hash/1, fnv/1, nthdelete/2, zero_split/1, nthreplace/3, rand_str/1, position/2, shuffle/1, floor/1, ceiling/1, time_to_epoch_int/1, time_to_epoch_float/1, now_int/0, now_float/0, byte_size/1, listify/1, reverse_bits/1]). -ifdef(TEST). -include("etest/lib_misc_test.erl"). @@ -111,12 +111,13 @@ gather(N, Max, Ref, L) -> {Ref, Ret} -> gather(N-1, Max, Ref, [Ret|L]) end. +hash(Term) -> fnv:hash(Term). %32 bit fnv. magic numbers ahoy -hash(Term) when is_binary(Term) -> +fnv(Term) when is_binary(Term) -> fnv_int(?OFFSET_BASIS, 0, Term); -hash(Term) -> +fnv(Term) -> fnv_int(?OFFSET_BASIS, 0, term_to_binary(Term)). fnv_int(Hash, ByteOffset, Bin) when byte_size(Bin) == ByteOffset -> elibs/lib_misc.erl @@ -28,4 +28,35 @@ zero_split_test() -> ?assertEqual({<<"">>, <<0,"abcdefg">>}, zero_split(<<0, "abcdefg">>)), ?assertEqual({<<"abd">>, <<0, "efg">>}, zero_split(<<"abd", 0, "efg">>)), ?assertEqual({<<"abcdefg">>, <<0>>}, zero_split(<<"abcdefg",0>>)), - ?assertEqual(<<"abcdefg">>, zero_split(<<"abcdefg">>)). \ No newline at end of file + ?assertEqual(<<"abcdefg">>, zero_split(<<"abcdefg">>)). + + +hash_throughput_test_() -> + {timeout, 120, [{?LINE, fun() -> + Keys = lists:map(fun(N) -> + lists:duplicate(1000, random:uniform(255)) + end, lists:seq(1,1000)), + FNVStart = now_float(), + lists:foreach(fun(Key) -> + fnv(Key) + end, Keys), + FNVEnd = now_float(), + ?debugFmt("fnv took ~ps~n", [FNVEnd - FNVStart]), + MStart = now_float(), + lists:foreach(fun(Key) -> + hash(Key) + end, Keys), + MEnd = now_float(), + ?debugFmt("murmur took ~ps~n", [MEnd - MStart]), + FNVNStart = now_float(), + lists:foreach(fun(Key) -> + fnv:hash(Key) + end, Keys), + FNVNEnd = now_float(), + ?debugFmt("fnv native took ~ps~n", [FNVNEnd - FNVNStart]) + end}]}. + +fnv_native_compat_test() -> + ?assertEqual(fnv("blah"), fnv:hash("blah")), + ?assertEqual(fnv(<<"blah">>), fnv:hash(<<"blah">>)), + ?assertEqual(fnv([<<"blah">>, "bleg"]), fnv:hash([<<"blah">>, "bleg"])). \ No newline at end of file etest/lib_misc_test.erl 7a41a657cf66acc597409c5d604939d9d26ea0df Cliff Moon cliff@moonpolysoft.com http://github.com/cliffmoon/dynomite/commit/fecb101934641a358a39fdaf83b5ac47cf7b942a fecb101934641a358a39fdaf83b5ac47cf7b942a 2009-01-31T14:13:47-08:00 2009-01-31T14:13:47-08:00 added native murmur and fnv hashing. ce9c951834a1692c3169de70e91c0ddf617c5b74 Cliff Moon cliff@moonpolysoft.com