From 31e3cf9cc14bbde0d82de181e9e05aa8bd8fc2d2 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Sun, 26 Apr 2009 12:36:27 -0700
Subject: [PATCH 01/15] get rid of bus error for type

---
 c/bloom_drv.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/c/bloom_drv.c b/c/bloom_drv.c
index 3aa15d5..5eaa01f 100644
--- a/c/bloom_drv.c
+++ b/c/bloom_drv.c
@@ -87,14 +87,22 @@ static void output(ErlDrvData handle, char *buf, int len) {
 static void setup(bloom_drv_t *driver, char *buf, int len) {
   long n;
   double e;
+  char *filename;
+  int size;
+  int type;
   int index = 0;
   
   ei_decode_version(buf, &index, NULL);
   ei_decode_tuple_header(buf, &index, NULL);
+  ei_get_type(buf, &index, &type, &size);
+  filename = driver_alloc(size+1);
+  ei_decode_string(buf, &index, filename);
   ei_decode_long(buf, &index, &n);
   ei_decode_double(buf, &index, &e);
   
-  driver->bloom = bloom_create(n, e);
+  driver->bloom = bloom_open(filename, n, e);
+  
+  driver_free(filename);
 }
 
 static void put(bloom_drv_t *driver, char *buf, int len) {

From 35776ac73db9489d5d4d4c77fe3b9a6d08a005f3 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Sun, 26 Apr 2009 12:37:11 -0700
Subject: [PATCH 02/15] tests for new interface

---
 c/bloom.c            |  2 ++
 elibs/bloom.erl      |  6 +++---
 etest/bloom_test.erl | 19 ++++++++++++++++---
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/c/bloom.c b/c/bloom.c
index 6344a0b..fa05710 100644
--- a/c/bloom.c
+++ b/c/bloom.c
@@ -31,6 +31,7 @@ bloom_t *bloom_open(char* filename, long n, double e) {
   
   if (-1 == stat(filename, &file_stat)) {
     //create a new one
+    // printf("creating new file\n");
     if (-1 == (file = open(filename, O_CREAT | O_RDWR, S_IWUSR | S_IRUSR | S_IRGRP | S_IWGRP))) {
       return NULL;
     }
@@ -46,6 +47,7 @@ bloom_t *bloom_open(char* filename, long n, double e) {
     bloom->data.k = (int) round(log(2) * m / n);
     pwrite(file, bloom, sizeof(bloom_t) + BYTE_SIZE(m), 0);
   } else {
+    // printf("opening existing file\n");
     if (-1 == (file = open(filename, O_RDWR))) {
       return NULL;
     }
diff --git a/elibs/bloom.erl b/elibs/bloom.erl
index 18d312c..395d554 100644
--- a/elibs/bloom.erl
+++ b/elibs/bloom.erl
@@ -12,7 +12,7 @@
 -author('cliff@powerset.com').
 
 %% API
--export([start/2, put/2, has/2, mem_size/1, key_size/1, stop/1]).
+-export([start/3, put/2, has/2, mem_size/1, key_size/1, stop/1]).
 
 %% COMMANDS
 -define(SETUP, $s).
@@ -34,11 +34,11 @@
 %% @end 
 %%--------------------------------------------------------------------
 
-start(N, E) ->
+start(Filename, N, E) ->
   case load_driver() of
     ok ->
       P = open_port({spawn, 'bloom_drv'}, [binary]),
-      port_command(P, [?SETUP, term_to_binary({N, E})]),
+      port_command(P, [?SETUP, term_to_binary({Filename, N, E})]),
       {ok, {bloom, P}};
     {error, Err} ->
       Msg = erl_ddll:format_error(Err),
diff --git a/etest/bloom_test.erl b/etest/bloom_test.erl
index cad261b..67bf97e 100644
--- a/etest/bloom_test.erl
+++ b/etest/bloom_test.erl
@@ -1,14 +1,16 @@
 -include_lib("eunit/include/eunit.hrl").
 
 simple_bloom_test() ->
-  {ok, Bloom} = bloom:start(10000, 0.001),
+  file:delete(data_file()),
+  {ok, Bloom} = bloom:start(data_file(), 10000, 0.001),
   bloom:put(Bloom, "wut"),
   ?assertEqual(true, bloom:has(Bloom, "wut")),
   ?assertEqual(false, bloom:has(Bloom, "fuck")),
   bloom:stop(Bloom).
 
 insert_many_things_test() ->
-  {ok, Bloom} = bloom:start(10000, 0.001),
+  file:delete(data_file()),
+  {ok, Bloom} = bloom:start(data_file(), 10000, 0.001),
   Keys = lists:map(fun(N) ->
       Key = "Key" ++ float_to_list(random:uniform()),
       bloom:put(Bloom, Key),
@@ -20,7 +22,8 @@ insert_many_things_test() ->
   bloom:stop(Bloom).
 
 false_positive_error_rate_test() ->
-  {ok, Bloom} = bloom:start(10000, 0.001),
+  file:delete(data_file()),
+  {ok, Bloom} = bloom:start(data_file(), 10000, 0.001),
   lists:foreach(fun(N) ->
       Key = "Key" ++ float_to_list(random:uniform()),
       bloom:put(Bloom, Key)
@@ -34,3 +37,13 @@ false_positive_error_rate_test() ->
   ?assertEqual(10000, bloom:key_size(Bloom)),
   bloom:stop(Bloom).
   
+priv_dir() ->
+  Dir = filename:join(t:config(priv_dir), "data"),
+  filelib:ensure_dir(filename:join(Dir, "bloom")),
+  Dir.
+
+data_file() ->
+  filename:join(priv_dir(), "bloom").
+
+data_file(N) ->
+  filename:join(priv_dir(), "bloom" ++ integer_to_list(N)).
\ No newline at end of file

From 22d4215b7fbf278fbaa4e32becc23a717225cc68 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Sun, 26 Apr 2009 12:37:31 -0700
Subject: [PATCH 03/15] remove couch test

---
 etest/storage_server_test.erl | 38 +++++++++++++++++------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/etest/storage_server_test.erl b/etest/storage_server_test.erl
index 7a99296..84e320f 100644
--- a/etest/storage_server_test.erl
+++ b/etest/storage_server_test.erl
@@ -34,25 +34,25 @@ test_storage_server_throughput() ->
   ?debugFmt("storage server can do ~p reqs/s", [20000/(End-Start)]),
   storage_server:close(Pid).
 
-couch_storage_test() ->
-    configuration:start_link(#config{}),
-    CouchFile = filename:join(priv_dir(), "couch"),
-    {ok, State} = couch_storage:open(CouchFile, storage_test),
-    {ok, St2} = couch_storage:put("key_one", context, <<"value one">>, State),
-    {ok, St3} = couch_storage:put("key_one", context, <<"value one">>, St2),
-    {ok, St4} = couch_storage:put("key_two", context, <<"value two">>, St3),
-    Result = couch_storage:fold(fun({Key, Context, Value}, Acc) -> [Key|Acc] end, St4, []),
-    timer:sleep(100),
-    ["key_two", "key_one"] = Result,
-    {ok, {context, <<"value one">>}} = couch_storage:get("key_one", St4),
-    {ok, true} = couch_storage:has_key("key_one", St4),
-    {ok, St5} = couch_storage:delete("key_one", St4),
-    {ok, false} = couch_storage:has_key("key_one", St5),
-    {ok, true} = couch_storage:has_key("key_two", St5),
-    {ok, St6} = couch_storage:delete("key_two", St5),
-    configuration:stop(),
-    timer:sleep(1),
-    couch_storage:close(St6).
+% couch_storage_test() ->
+%     configuration:start_link(#config{}),
+%     CouchFile = filename:join(priv_dir(), "couch"),
+%     {ok, State} = couch_storage:open(CouchFile, storage_test),
+%     {ok, St2} = couch_storage:put("key_one", context, <<"value one">>, State),
+%     {ok, St3} = couch_storage:put("key_one", context, <<"value one">>, St2),
+%     {ok, St4} = couch_storage:put("key_two", context, <<"value two">>, St3),
+%     Result = couch_storage:fold(fun({Key, Context, Value}, Acc) -> [Key|Acc] end, St4, []),
+%     timer:sleep(100),
+%     ["key_two", "key_one"] = Result,
+%     {ok, {context, <<"value one">>}} = couch_storage:get("key_one", St4),
+%     {ok, true} = couch_storage:has_key("key_one", St4),
+%     {ok, St5} = couch_storage:delete("key_one", St4),
+%     {ok, false} = couch_storage:has_key("key_one", St5),
+%     {ok, true} = couch_storage:has_key("key_two", St5),
+%     {ok, St6} = couch_storage:delete("key_two", St5),
+%     configuration:stop(),
+%     timer:sleep(1),
+%     couch_storage:close(St6).
 
 dict_storage_test() ->
     configuration:start_link(#config{}),

From b07c67c8ec3d5b8df92169aff4b4e2334809a2e1 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Sun, 26 Apr 2009 18:44:36 -0700
Subject: [PATCH 04/15] fix persistence offset issues

---
 c/bloom.c            | 27 +++++++++++++++++++++++----
 c/bloom.h            | 12 ++++++------
 etest/bloom_test.erl | 27 +++++++++++++++++++++++++++
 3 files changed, 56 insertions(+), 10 deletions(-)

diff --git a/c/bloom.c b/c/bloom.c
index fa05710..cf138cb 100644
--- a/c/bloom.c
+++ b/c/bloom.c
@@ -29,6 +29,8 @@ bloom_t *bloom_open(char* filename, long n, double e) {
   uint32_t version;
   struct stat file_stat;
   
+  // printf("sizeof(bloom_data_t) %d\n", sizeof(bloom_data_t));
+  
   if (-1 == stat(filename, &file_stat)) {
     //create a new one
     // printf("creating new file\n");
@@ -45,7 +47,7 @@ bloom_t *bloom_open(char* filename, long n, double e) {
     
     bloom->data.m = m;
     bloom->data.k = (int) round(log(2) * m / n);
-    pwrite(file, bloom, sizeof(bloom_t) + BYTE_SIZE(m), 0);
+    pwrite(file, &bloom->data, sizeof(bloom_data_t) + BYTE_SIZE(m), 0);
   } else {
     // printf("opening existing file\n");
     if (-1 == (file = open(filename, O_RDWR))) {
@@ -53,10 +55,23 @@ bloom_t *bloom_open(char* filename, long n, double e) {
     }
     
     pread(file, &version, sizeof(uint32_t), 0);
-    pread(file, &m, sizeof(uint32_t), sizeof(uint32_t));
+    pread(file, &m, sizeof(uint32_t), 4);
+    
+    // printf("read version of %d\n", version);
+    // printf("read m of %d\n", m);
     bloom = malloc(sizeof(bloom_t) + BYTE_SIZE(m));
     pread(file, &bloom->data, sizeof(bloom_data_t) + BYTE_SIZE(m), 0);
   }
+  // printf("bloom->data %d\n", (int)&bloom->data);
+  // printf("n %d\n", (int)&bloom->data.n);
+  // printf("keys %d\n", (int)&bloom->data.keys);
+  // printf("version = %d %d\n", bloom->data.version, (( int)&(bloom->data.version) - ( int)&(bloom->data)));
+  // printf("m = %d %d\n", bloom->data.m, (( int)&(bloom->data.m) - ( int)&(bloom->data)));
+  // printf("n = %d %d\n", bloom->data.n, (( int)&(bloom->data.n) - ( int)&(bloom->data)));
+  // printf("e = %f %d\n", bloom->data.e, (( int)&(bloom->data.e) - ( int)&(bloom->data)));
+  // printf("k = %d %d\n", bloom->data.k, (( int)&(bloom->data.k) - ( int)&(bloom->data)));
+  // printf("keys = %d %d\n", bloom->data.keys, (( int)&(bloom->data.keys) - ( int)&(bloom->data)));
+  // printf("seed = %d %d\n", bloom->data.seed, (( int)&(bloom->data.seed) - ( int)&(bloom->data)));
   bloom->file = file;
   bloom->filename = malloc(strlen(filename) + 1);
   strcpy(bloom->filename, filename);
@@ -79,11 +94,15 @@ void bloom_put(bloom_t *bloom, char *buff, int len) {
     // printf("byte %d bit %d\n", BYTE_INDEX(index), BIT_INDEX(index));
     byte_index = BYTE_INDEX(index);
     SET_BIT(bloom->data.bits, index);
-    pwrite(bloom->file, &bloom->data.bits[byte_index], 1, sizeof(bloom_t) + byte_index - 1);
+    // if ((sizeof(bloom_data_t) + byte_index - 1) < 108) {
+    //   printf("writing to %d\n", sizeof(bloom_data_t) + byte_index - 1);
+    offset = (unsigned int)&(bloom->data.bits[byte_index]) - (unsigned int)&bloom->data;
+    pwrite(bloom->file, &bloom->data.bits[byte_index], 1, offset);
     // printf("byte %d\n", bloom->bits[BYTE_INDEX(index)]);
   }
   bloom->data.keys++;
-  offset = ((unsigned int)&(bloom->data.keys) - (unsigned int)bloom);
+  offset = ((unsigned int)&(bloom->data.keys) - (unsigned int)&(bloom->data));
+  // printf("writing keys into offset %d\n", offset);
   pwrite(bloom->file, &(bloom->data.keys), sizeof(uint32_t), offset);
 }
 
diff --git a/c/bloom.h b/c/bloom.h
index 827d968..cc8c863 100644
--- a/c/bloom.h
+++ b/c/bloom.h
@@ -1,11 +1,11 @@
 
 typedef struct _bloom_data_t {
-  uint32_t version;
-  uint32_t m;
-  uint64_t n;
-  double e;
-  uint32_t k;
-  uint64_t keys;
+  uint32_t version; //0
+  uint32_t m;       //4
+  uint64_t n;       //8
+  double e;         //
+  uint32_t k;       //
+  uint64_t keys;    //3
   uint32_t seed;
   char reserved[64];
   char bits[1];
diff --git a/etest/bloom_test.erl b/etest/bloom_test.erl
index 67bf97e..98e0c00 100644
--- a/etest/bloom_test.erl
+++ b/etest/bloom_test.erl
@@ -37,6 +37,33 @@ false_positive_error_rate_test() ->
   ?assertEqual(10000, bloom:key_size(Bloom)),
   bloom:stop(Bloom).
   
+persist_test() ->
+  file:delete(data_file()),
+  {ok, Bloom} = bloom:start(data_file(), 10000, 0.001),
+  Keys = lists:map(fun(N) ->
+      Key = "Key" ++ float_to_list(random:uniform()),
+      bloom:put(Bloom, Key),
+      Key
+    end, lists:seq(1, 10000)),
+  ?debugMsg("got keys"),
+  bloom:stop(Bloom),
+  ?debugMsg("stopping bloom"),
+  {ok, Bloom2} = bloom:start(data_file(), 10000, 0.001),
+  ?debugMsg("restarted"),
+  FalsePositives = [X || X <- [bloom:has(Bloom2, "butt" ++ float_to_list(random:uniform())) || N <- lists:seq(1,10000)], X == true],
+  FPRate = length(FalsePositives) / 10000,
+  ?debugFmt("false positives: ~p", [length(FalsePositives)]),
+  ?debugFmt("false positives: ~p", [FPRate]),
+  ?debugFmt("mem size ~p", [bloom:mem_size(Bloom2)]),
+  ?assert(FPRate < 0.001),
+  ?assertEqual(10000, bloom:key_size(Bloom2)),
+  TruePositives = [X || X <- [bloom:has(Bloom2, Key) || Key <- Keys], X == true],
+  ?debugFmt("true positives: ~p", [length(TruePositives)]),
+  ?debugFmt("keys ~p", [length(Keys)]),
+  TPRate = length(TruePositives) / 10000,
+  ?assertEqual(1.0, TPRate),
+  bloom:stop(Bloom2).
+  
 priv_dir() ->
   Dir = filename:join(t:config(priv_dir), "data"),
   filelib:ensure_dir(filename:join(Dir, "bloom")),

From 949683421b9724c9aead381f9f78da41856f0b05 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Sun, 26 Apr 2009 20:06:16 -0700
Subject: [PATCH 05/15] fix up the console

---
 elibs/web_rpc.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/elibs/web_rpc.erl b/elibs/web_rpc.erl
index ed2db49..f61c93f 100644
--- a/elibs/web_rpc.erl
+++ b/elibs/web_rpc.erl
@@ -63,12 +63,12 @@ rates(Node) ->
 syncs_running(cluster) ->
   {Good,_} = rpc:multicall(sync_manager, running, []),
   {array, lists:map(fun({Part, NodeA, NodeB}) -> 
-      {struct, [{partition, Part}, {nodes, [NodeA, NodeB]}]}
+      {struct, [{partition, Part}, {nodes, {array, [NodeA, NodeB]}}]}
     end, lists:flatten(Good))};
   
 syncs_running(Node) ->
   {array, lists:map(fun({Part, NodeA, NodeB}) -> 
-      {struct, [{partition, Part}, {nodes, [NodeA, NodeB]}]}
+      {struct, [{partition, Part}, {nodes, {array, [NodeA, NodeB]}}]}
     end, sync_manager:running(Node))}.
 
 diff_size(cluster) ->

From cb51eb8ae89c8b5404ec35f3e028b248e795bdc8 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Mon, 27 Apr 2009 13:49:46 -0700
Subject: [PATCH 06/15] refactor left thrift beams out in the cold

---
 Rakefile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/Rakefile b/Rakefile
index acc0203..62c4dc1 100644
--- a/Rakefile
+++ b/Rakefile
@@ -213,6 +213,7 @@ end
 DRIVERS = FileList['c/*_drv.c'].pathmap("%{c,priv}X.so")
 BEAMS = FileList['elibs/*.erl'].pathmap("%{elibs,ebin}X.beam")
 TEST_BEAMS = FileList['etest/*.erl'].select {|d| d !~ /^.*_test.erl$/}.pathmap("%{etest,ebin}X.beam")
+THRIFT_BEAMS = FileList['gen-erl/*.erl'].pathmap("%{gen-erl,ebin}X.beam")
 
 directory "build"
 directory "priv"
@@ -249,5 +250,9 @@ rule ".beam" => "%{ebin,etest}X.erl" do |t|
   compile(t)
 end
 
+rule ".beam" => "%{ebin,gen-erl}X.erl" do |t|
+  compile(t)
+end
+
 task :build_c_drivers => [:c_env, "priv"] + DRIVERS
-task :build_erl => BEAMS + TEST_BEAMS
+task :build_erl => BEAMS + TEST_BEAMS + THRIFT_BEAMS

From 89cd8f0f4604891cd0dbdba21af4eddaefba10aa Mon Sep 17 00:00:00 2001
From: Alexander Staubo <alex@bengler.no>
Date: Wed, 29 Apr 2009 04:43:10 +0800
Subject: [PATCH 07/15] Fix the status command's usage line to reflect the
 command name.

Signed-off-by: Cliff Moon <cliff@moonpolysoft.com>
---
 rlibs/cli/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rlibs/cli/status.rb b/rlibs/cli/status.rb
index 5d2bcd6..fd4cd90 100644
--- a/rlibs/cli/status.rb
+++ b/rlibs/cli/status.rb
@@ -1,7 +1,7 @@
 options = {}
 
 OptionParser.new do |opts|
-  opts.banner = "Usage: dynomite stop [options]"
+  opts.banner = "Usage: dynomite status [options]"
 
   contents =  File.read(File.dirname(__FILE__) + "/shared/common.rb")
   eval contents

From b13b1700c82df085b3cbf98465a74c747bdb030d Mon Sep 17 00:00:00 2001
From: Alexander Staubo <alex@bengler.no>
Date: Wed, 29 Apr 2009 04:43:37 +0800
Subject: [PATCH 08/15] The status command should use the -o parameter in the
 same way as the start command.

Signed-off-by: Cliff Moon <cliff@moonpolysoft.com>
---
 rlibs/cli/status.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rlibs/cli/status.rb b/rlibs/cli/status.rb
index fd4cd90..b06d47e 100644
--- a/rlibs/cli/status.rb
+++ b/rlibs/cli/status.rb
@@ -9,6 +9,6 @@
 
 cookie = Digest::MD5.hexdigest(options[:cluster] + "NomMxnLNUH8suehhFg2fkXQ4HVdL2ewXwM")
 
-str = %Q(erl -smp -sname console_#{$$} -hidden -setcookie #{cookie} -pa #{ROOT}/ebin/ -run commands start -run erlang halt -noshell -node #{options[:name]}@#{`hostname -s`.chomp} -m membership -f status)
+str = %Q(erl -smp -sname console_#{$$} -hidden -setcookie #{cookie} -pa #{ROOT}/ebin/ -run commands start -run erlang halt -noshell -node #{options[:name]} -m membership -f status)
 puts str
 exec str

From 40ef3daf90ac4770d327d906010472c09811a168 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 08:38:42 -0700
Subject: [PATCH 09/15] more aggressive GC in dmerkle and set the key diff
 timeout to infinity

---
 elibs/dmerkle.erl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/elibs/dmerkle.erl b/elibs/dmerkle.erl
index 64228d2..e70c32c 100644
--- a/elibs/dmerkle.erl
+++ b/elibs/dmerkle.erl
@@ -37,7 +37,7 @@ open(FileName) ->
   open(FileName, undefined).
 
 open(FileName, BlockSize) -> 
-  gen_server:start_link(?MODULE, {FileName,BlockSize}, []).
+  gen_server:start_link(?MODULE, {FileName,BlockSize}, [{spawn_opt, [fullsweep_after, 10]}]).
 
 count_trace(Pid, Key) ->
   gen_server:call(Pid, {count_trace, Key}).
@@ -70,7 +70,7 @@ deletea(Key, Tree) ->
   gen_server:cast(Tree, {delete, Key}).
 
 key_diff(TreeA, TreeB) ->
-  gen_server:call(TreeA, {key_diff, TreeB}).
+  gen_server:call(TreeA, {key_diff, TreeB}, infinity).
 
 close(Tree) ->
   gen_server:cast(Tree, close).

From 74ded87d16e2f96aaadf52c14ba725d89cb16fa1 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 08:39:16 -0700
Subject: [PATCH 10/15] inifinite timeouts for the reload functions

---
 elibs/storage_manager.erl | 4 ++--
 elibs/sync_manager.erl    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/elibs/storage_manager.erl b/elibs/storage_manager.erl
index 4711657..97031f8 100644
--- a/elibs/storage_manager.erl
+++ b/elibs/storage_manager.erl
@@ -41,10 +41,10 @@ start_link() ->
   gen_server:start_link({local, storage_manager}, ?MODULE, [], []).
   
 load(Nodes, Partitions, PartsForNode) ->
-  gen_server:call(storage_manager, {load, Nodes, Partitions, PartsForNode, true}).
+  gen_server:call(storage_manager, {load, Nodes, Partitions, PartsForNode, true}, infinity).
   
 load_no_boot(Nodes, Partitions, PartsForNode) ->
-  gen_server:call(storage_manager, {load, Nodes, Partitions, PartsForNode, false}).
+  gen_server:call(storage_manager, {load, Nodes, Partitions, PartsForNode, false}, infinity).
   
 loaded() ->
   gen_server:call(storage_manager, loaded).
diff --git a/elibs/sync_manager.erl b/elibs/sync_manager.erl
index df62fa5..99d2ebb 100644
--- a/elibs/sync_manager.erl
+++ b/elibs/sync_manager.erl
@@ -41,7 +41,7 @@ stop() ->
   gen_server:cast(sync_manager, stop).
 
 load(Nodes, Partitions, PartsForNode) ->
-  gen_server:call(sync_manager, {load, Nodes, Partitions, PartsForNode}).
+  gen_server:call(sync_manager, {load, Nodes, Partitions, PartsForNode}, infinity).
 
 sync(Part, Master, NodeA, NodeB, DiffSize) ->
   gen_server:cast(sync_manager, {sync, Part, Master, NodeA, NodeB, DiffSize}).

From ce4e89737211eeb637b7583f1aba8e9ceeeefdc7 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 08:40:32 -0700
Subject: [PATCH 11/15] more aggressive GC in the storage server

---
 elibs/storage_server.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elibs/storage_server.erl b/elibs/storage_server.erl
index 85340fe..a14765c 100644
--- a/elibs/storage_server.erl
+++ b/elibs/storage_server.erl
@@ -41,7 +41,7 @@
 %% @end 
 %%--------------------------------------------------------------------
 start_link(StorageModule, DbKey, Name, Min, Max, BlockSize) when is_list(StorageModule) ->
-  gen_server:start_link({local, Name}, ?MODULE, {list_to_atom(StorageModule),DbKey,Name,Min,Max, BlockSize}, []);
+  gen_server:start_link({local, Name}, ?MODULE, {list_to_atom(StorageModule),DbKey,Name,Min,Max, BlockSize}, [{spawn_opt, [fullsweep_after, 10]}]);
 
 start_link(StorageModule, DbKey, Name, Min, Max, BlockSize) ->
   gen_server:start_link({local, Name}, ?MODULE, {StorageModule,DbKey,Name,Min,Max, BlockSize}, []).

From 3b54e0a974289c8a550624bd583d8ca847d07965 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 08:40:57 -0700
Subject: [PATCH 12/15] use proc_lib to start the sync server

---
 elibs/sync_server.erl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/elibs/sync_server.erl b/elibs/sync_server.erl
index 031c6c8..751bc12 100644
--- a/elibs/sync_server.erl
+++ b/elibs/sync_server.erl
@@ -25,7 +25,7 @@
 %% @end 
 %%--------------------------------------------------------------------
 start_link(Name, Partition) ->
-  Pid = spawn_link(fun() ->
+  Pid = proc_lib:spawn_link(fun() ->
       sync_server:loop(#state{name=Name,partition=Partition,paused=false})
     end),
   register(Name, Pid),

From 1c9a48d82d129108e3cb5f93e0a848154291802f Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 08:45:44 -0700
Subject: [PATCH 13/15] correct options for GC

---
 elibs/dmerkle.erl        | 2 +-
 elibs/storage_server.erl | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/elibs/dmerkle.erl b/elibs/dmerkle.erl
index e70c32c..1d9507b 100644
--- a/elibs/dmerkle.erl
+++ b/elibs/dmerkle.erl
@@ -37,7 +37,7 @@ open(FileName) ->
   open(FileName, undefined).
 
 open(FileName, BlockSize) -> 
-  gen_server:start_link(?MODULE, {FileName,BlockSize}, [{spawn_opt, [fullsweep_after, 10]}]).
+  gen_server:start_link(?MODULE, {FileName,BlockSize}, [{spawn_opt, [{fullsweep_after, 10}]}]).
 
 count_trace(Pid, Key) ->
   gen_server:call(Pid, {count_trace, Key}).
diff --git a/elibs/storage_server.erl b/elibs/storage_server.erl
index a14765c..005a7f5 100644
--- a/elibs/storage_server.erl
+++ b/elibs/storage_server.erl
@@ -41,7 +41,7 @@
 %% @end 
 %%--------------------------------------------------------------------
 start_link(StorageModule, DbKey, Name, Min, Max, BlockSize) when is_list(StorageModule) ->
-  gen_server:start_link({local, Name}, ?MODULE, {list_to_atom(StorageModule),DbKey,Name,Min,Max, BlockSize}, [{spawn_opt, [fullsweep_after, 10]}]);
+  gen_server:start_link({local, Name}, ?MODULE, {list_to_atom(StorageModule),DbKey,Name,Min,Max, BlockSize}, [{spawn_opt, [{fullsweep_after, 10}]}]);
 
 start_link(StorageModule, DbKey, Name, Min, Max, BlockSize) ->
   gen_server:start_link({local, Name}, ?MODULE, {StorageModule,DbKey,Name,Min,Max, BlockSize}, []).

From 43cd00811d70e2ce0bebb6a87f86165f1c92cef7 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Wed, 29 Apr 2009 09:24:19 -0700
Subject: [PATCH 14/15] get rid of couch stuff in appfile

---
 ebin/dynomite.app | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ebin/dynomite.app b/ebin/dynomite.app
index f8e67e9..d03ac94 100644
--- a/ebin/dynomite.app
+++ b/ebin/dynomite.app
@@ -4,7 +4,7 @@
    {vsn, "?VERSION"},
    {modules, 
    [
-      block_server, bootstrap, commands, configuration, couch_btree, couch_file, dets_storage, dict_storage, dmerkle,
+      block_server, bootstrap, commands, configuration, dets_storage, dict_storage, dmerkle,
       dmtree, dynomite_pb, dummy_server, dynomite, dynomite_app, dynomite_prof, dynomite_sup, dynomite_thrift_service, dynomite_web, fail_storage, fnv, fs_storage,
       lib_misc, mediator, membership, mnesia_storage, murmur, partitions, rate, socket_server, stats_server, storage_manager,
       storage_server, storage_server_sup, stream, sync_manager, sync_server, sync_server_sup, tc_storage, ulimit, vector_clock,

From 67e139f73dd06a7907296c157c14f4fc0a859ad4 Mon Sep 17 00:00:00 2001
From: Cliff Moon <cliff@moonpolysoft.com>
Date: Fri, 1 May 2009 10:08:17 -0700
Subject: [PATCH 15/15] robustify the rpc module against node failure

---
 elibs/dynomite.erl     | 11 +++----
 elibs/dynomite_rpc.erl | 65 ++++++++++++++++++++++++++++++------------
 elibs/dynomite_sup.erl |  2 +-
 3 files changed, 54 insertions(+), 24 deletions(-)

diff --git a/elibs/dynomite.erl b/elibs/dynomite.erl
index 22bdebc..e021a1b 100644
--- a/elibs/dynomite.erl
+++ b/elibs/dynomite.erl
@@ -8,17 +8,18 @@ start() ->
   crypto:start(),
   load_and_start_apps([os_mon, thrift, mochiweb, dynomite]).
   
-running(Node) when Node == node() ->
-  true;
+% running(Node) when Node == node() ->
+%   true;
   
 running(Node) ->
   Ref = erlang:monitor(process, {membership, Node}),
-  receive
+  R = receive
     {'DOWN', Ref, _, _, _} -> false
   after 1 ->
-    erlang:demonitor(Ref),
     true
-  end.
+  end,
+  erlang:demonitor(Ref),
+  R.
   
 running_nodes() ->
   [Node || Node <- nodes([this,visible]), dynomite:running(Node)].
diff --git a/elibs/dynomite_rpc.erl b/elibs/dynomite_rpc.erl
index 78b2749..d25a307 100644
--- a/elibs/dynomite_rpc.erl
+++ b/elibs/dynomite_rpc.erl
@@ -7,32 +7,61 @@
 
 connect(Node) ->
   case net_adm:ping(Node) of
-    pong -> {ok, Node};
+    pong ->
+      {ok, Node};
     pang -> {error, "Cannot connect."}
   end.
   
 get(Node, Key) ->
-  case rpc:call(Node, mediator, get, [Key]) of
-    {badrpc, Reason} -> {failure, Reason};
-    Result -> Result
-  end.
+  GetFun = fun(N) ->
+    case rpc:call(N, mediator, get, [Key]) of
+      {badrpc, Reason} -> {failure, Reason};
+      Result -> Result
+    end
+  end,
+  robustify(Node, GetFun).
   
 put(Node, Key, Context, Value) ->
-  case rpc:call(Node, mediator, put, [Key, Context, Value]) of
-    {badrpc, Reason} -> {failure, Reason};
-    Result -> Result
-  end.
+  PutFun = fun(N) ->
+    case rpc:call(N, mediator, put, [Key, Context, Value]) of
+      {badrpc, Reason} -> {failure, Reason};
+      Result -> Result
+    end
+  end,
+  robustify(Node, PutFun).
   
 has_key(Node, Key) ->
-  case rpc:call(Node, mediator, has_key, [Key]) of
-    {badrpc, Reason} -> {failure, Reason};
-    Result -> Result
-  end.
+  HasFun = fun(N) ->
+    case rpc:call(N, mediator, has_key, [Key]) of
+      {badrpc, Reason} -> {failure, Reason};
+      Result -> Result
+    end
+  end,
+  robustify(Node, HasFun).
   
 delete(Node, Key) ->
-  case rpc:call(Node, mediator, delete, [Key]) of
-    {badrpc, Reason} -> {failure, Reason};
-    Result -> Result
-  end.
+  DelFun = fun(N) ->
+    case rpc:call(N, mediator, delete, [Key]) of
+      {badrpc, Reason} -> {failure, Reason};
+      Result -> Result
+    end
+  end,
+  robustify(Node, DelFun).
   
-close(Node) -> erlang:disconnect(Node).
\ No newline at end of file
+close(Node) -> erlang:disconnect(Node).
+
+
+robustify(Node, Fun) ->
+  erlang:monitor_node(Node, true),
+  R = receive
+    {nodedown, Node} ->
+      % io:format("node ~p was down~n", [Node]),
+      case dynomite:running_nodes() of
+        [] -> {failure, "No dynomite nodes available."};
+        [NextNode|_] -> Fun(NextNode)
+      end
+  after 0 ->
+    Fun(Node)
+  end,
+  erlang:monitor_node(Node, false),
+  R.
\ No newline at end of file
diff --git a/elibs/dynomite_sup.erl b/elibs/dynomite_sup.erl
index 131302d..e37bbef 100644
--- a/elibs/dynomite_sup.erl
+++ b/elibs/dynomite_sup.erl
@@ -49,7 +49,7 @@ start_link(ConfigFile) ->
 %%--------------------------------------------------------------------
 init(ConfigFile) ->
     Node = node(),
-    Nodes = dynomite:running_nodes(),
+    Nodes = dynomite:running_nodes() ++ [node()],
     Children = [
                 {fnv,
                  {fnv,start,[]},