re-org of specs into local_store and server groups

chunks now track only their replication source and their replication client (only 1)

lib/monotable/client/server_client.rb

@@ -104,8 +104,11 @@ def rest_client_request(method,request_path,options={})
     #   https://github.com/archiloque/rest-client/blob/master/lib/restclient.rb
     def request(method,request_path,options={})
       request_path = "#{server}/#{request_path}"
-      return em_synchrony_request method, request_path, options if ServerClient.use_synchrony
-      rest_client_request(method,request_path,options)
+      if ServerClient.use_synchrony
+        em_synchrony_request method, request_path, options
+      else
+        rest_client_request(method,request_path,options)
+      end
     end
   end
 
@@ -174,29 +177,23 @@ def delete(key)
     end
   end
 
-  # this api is supported for testing, it should never be used by an actual client
-  module ServerClientServerReadAPI
+  # this api is supported for internal use, it should never be used by an actual client
+  module ServerClientServerAPI
     def chunks; request(:get,"server/chunks")[:chunks]; end
     def servers; request(:get,"server/servers")[:servers]; end
+    def local_store_status; request(:get,"server/local_store_status"); end
 
     # returns nil if chunk not found
-    def chunk_info(key); request(:get,"server/chunk_info/#{ue key}", :accept_404=>true)[:chunk_info]; end
-
-    # returns nil if chunk not found
+    def chunk_status(key); request(:get,"server/chunk_status/#{ue key}", :accept_404=>true)[:status]; end
     def chunk_keys(key); request(:get,"server/chunk_keys/#{ue key}", :accept_404=>true)[:keys]; end
-
-    #
-    def local_store_status; request(:get,"server/local_store_status"); end
+    def chunk_replication(key); request(:get,"server/chunk_replication/#{ue key}", :accept_404=>true); end
 
     # returns true if the server is up and responding to the heartbeat
     def up?
       request(:get,"server/heartbeat")[:status]=="alive";
     rescue Errno::ECONNREFUSED => e
     end
-  end
 
-  # this api is supported for testing, it should never be used by an actual client
-  module ServerClientServerModifyAPI
     def split_chunk(on_key); request(:post,"server/split_chunk/#{ue on_key}")[:chunks]; end
     def balance; request(:post,"server/balance"); end
     def join(server,skip_servers=[]); request(:put,"server/join?server_name=#{ue server}&skip_servers=#{ue skip_servers.join(',')}")[:servers]; end
@@ -212,16 +209,18 @@ def journal_write(chunk,journal_write_string);
 
     # returns the raw chunk-file
     def chunk(chunk_key); request(:get,"server/chunk/#{ue chunk_key}",:raw_response => true); end
+
+    # tell server to clone a chunk from_server to its own local_store
     def clone_chunk(chunk_key,from_server); request(:post,"server/clone_chunk/#{ue chunk_key}",:params => {:from_server => from_server}); end
     def delete_chunk(chunk_key); request(:delete,"server/chunk/#{ue chunk_key}"); end
-    def set_chunk_replication_clients(chunk_key,clients); request(:post,"server/chunk_replication_clients/#{ue chunk_key}", :params => {:clients => clients.join(',')}); end
+
+    def set_chunk_replication(chunk_key,source="",client=""); request(:post,"server/chunk_replication/#{ue chunk_key}", :params => {:replication_source => source, :replication_client => client}); end
   end
 
   class ServerClient
     include ServerClientReadAPI
     include ServerClientWriteAPI
-    include ServerClientServerReadAPI
-    include ServerClientServerModifyAPI
+    include ServerClientServerAPI
     include RestClientHelper
     attr_accessor :server, :client_options
     attr_accessor :path_prefix

lib/monotable/local_store/chunk.rb

@@ -215,6 +215,10 @@ def init_chunk(options={})
       @record_count_on_disk = 0
     end
 
+    def key
+      range_start
+    end
+
     def valid_range?
       range_start < range_end
     end

lib/monotable/local_store/disk_chunk_base.rb

@@ -12,7 +12,20 @@ module Monotable
   class DiskChunkBase < Chunk
     attr_accessor :path_store
     attr_accessor :journal
-    attr_accessor :replication_clients
+    attr_accessor :replication_client, :replication_source
+
+    # "" is replaced with nil
+    def replication_client=(rc)
+      @replication_client = (rc && rc.length==0) ? nil : rc
+    end
+
+    # "" is replaced with nil
+    def replication_source=(rc)
+      @replication_source = (rc && rc.length==0) ? nil : rc
+    end
+
+    def master?; !replication_source; end
+    def slave?; !!replication_source; end
 
     class << self
       def reset_disk_chunks
@@ -59,9 +72,6 @@ def init_disk_chunk_base(options={})
       raise ":filename required" unless options[:filename]
       init_chunk(options)
 
-      # init to no replicas
-      @replication_clients = []
-
       @journal = options[:journal] || (path_store && path_store.journal) || Journal.new(options[:filename]+".testing_journal")
 
       init_from_disk
@@ -85,11 +95,12 @@ def chunk_file_data
     end
 
     def status
-      super.merge :replication_clients => replication_clients.collect {|a|a.to_s}
+      super.merge(:replication_client => replication_client && replication_client.to_s,
+        :replication_source => replication_source && replication_source.to_s)
     end
 
     def journal_write(encoded_journal_entry)
-      replication_clients.each {|rc| rc.journal_write self, encoded_journal_entry}
+      replication_client.journal_write self, encoded_journal_entry if replication_client
       journal.journal_write self, encoded_journal_entry
     end
 

lib/monotable/local_store/local_store.rb

@@ -185,7 +185,7 @@ def add_chunk_internal(chunk)
         "Chunk2: #{chunk.basename.inspect}\n" if chunks_by_basename[chunk.basename]
       raise "chunk does not have a path_store" unless chunk.path_store
       chunks_by_basename[chunk.basename] =
-      chunks[chunk.range_start] = chunk
+      chunks[chunk.key] = chunk
     end
     public
 
@@ -223,15 +223,12 @@ def add_chunk(chunk)
       add_chunk_internal disk_chunk
     end
 
-    def delete_chunk(chunk_id)
-      chunk = chunks[chunk_id]
+    def delete_chunk(chunk)
+      chunk = chunks[chunk] if chunk.kind_of? String
       raise "chunk does not exist: #{chunk_id.inspect}" unless chunk
 
-      path_store = chunk.path_store
-
-      path_store.delete_chunk(chunk)
-
-      chunks.delete(chunk_id)
+      chunk.path_store.delete_chunk(chunk)
+      chunks.delete(chunk.key)
     end
 
     def update_path_store(chunk)
@@ -272,7 +269,7 @@ def compact(options={})
     def verify_chunk_ranges
       last_key=nil
       chunks.each do |key,chunk|
-        raise "key=#{key.inspect} doesn'chunks_by_basename match chunk.range_start=#{chunk.range_start.inspect}" unless key==chunk.range_start
+        raise "key=#{key.inspect} doesn't match chunk.key=#{chunk.key.inspect}" unless key==chunk.key
         raise "consecutive range keys out of order last_key=#{last_key.inspect} chunk.range_start=#{chunk.range_start.inspect} chunk.range_end=#{chunk.range_end.inspect}" unless
           last_key==nil || last_key<=chunk.range_start
       end

spec/cache_spec.rb → spec/local_store_spec/cache_spec.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::Cache do
   include MonotableHelperMethods

spec/disk_chunk_base_spec.rb → spec/local_store_spec/disk_chunk_base_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::DiskChunkBase do
   include MonotableHelperMethods

spec/disk_chunk_spec.rb → spec/local_store_spec/disk_chunk_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::DiskChunk do
   include MonotableHelperMethods

spec/file_handle_spec.rb → spec/local_store_spec/file_handle_spec.rb

@@ -1,6 +1,6 @@
 require 'tmpdir'
 require 'fileutils'
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 module Monotable
 describe FileHandle do

spec/journal_spec.rb → spec/local_store_spec/journal_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 # TODO: test the case where we write to a chunk, start async compaction, write again while compaction is going, and
 # THEN compaction completes. Are we in a consistent state? I don't think the chunk.reset currently does the right thing.

spec/local_store_api_spec.rb → spec/local_store_spec/local_store_api_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::LocalStore do
   include MonotableHelperMethods

spec/local_store_multi_spec.rb → spec/local_store_spec/local_store_multi_spec.rb

@@ -1,15 +1,11 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
+#require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
 require 'find'
 
 describe Monotable::LocalStore do
   include MonotableHelperMethods
   include DaemonTestHelper
 
-  after(:all) do
-    cleanup
-  end
-
   def blank_server
     reset_temp_dir
     Monotable::Server.new(:store_paths=>[local_store_path],:initialize_new_store=>true)

spec/local_store_spec.rb → spec/local_store_spec/local_store_spec.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::LocalStore do
   include MonotableHelperMethods

spec/memory_chunk_spec.rb → spec/local_store_spec/memory_chunk_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::MemoryChunk do
   include MonotableHelperMethods

spec/path_store_balancer_spec.rb → spec/local_store_spec/path_store_balancer_spec.rb

@@ -1,5 +1,5 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
+require File.expand_path(File.join(File.dirname(__FILE__),"..","daemon_test_helper"))
 require 'find'
 
 describe Monotable::PathStoreBalancer do

spec/string_spec.rb → spec/local_store_spec/string_spec.rb

@@ -1,5 +1,5 @@
 # encoding: BINARY
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::StringBinaryEnumeration do
 

spec/tools_spec.rb → spec/local_store_spec/tools_spec.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 module Monotable
 describe Tools do

spec/mono_table_helper_methods.rb

@@ -2,6 +2,8 @@
 #SimpleCov.start
 #$stderr.puts "SimpleCov.start"
 require File.join(File.dirname(__FILE__),"../lib/monotable/monotable")
+require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),'daemon_test_helper')
 
 class VirtualSizeFileSystemMock < Monotable::Tools::FileSystem
   def initialize(virtual_size)
@@ -58,7 +60,7 @@ def setup_store_with_test_keys(num_records=5)
   end
 
   def load_test_data(filename)
-    File.open(File.join(File.dirname(__FILE__),"test_data",filename)) {|f| return f.read.force_encoding("BINARY")}
+    File.open(File.join(test_data_dir,filename)) {|f| return f.read.force_encoding("BINARY")}
     nil
   end
 
@@ -88,9 +90,7 @@ def load_test_data_directory(target=nil,key_prefix=nil)
   def chunkify_test_data_directory
     temp_dir=reset_temp_dir
 
-    out_file=File.join(temp_dir,"test_data")
-
-    Monotable::Tools.chunkify_directory(test_data_dir,out_file)
+    Monotable::Tools.chunkify_directory(test_data_dir,temp_dir)
   end
 end
 

spec/2daemon_spec.rb → spec/server_spec/2daemon_spec.rb

@@ -1,12 +1,11 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 require 'rubygems'
 require 'rest_client'
 require 'tmpdir'
 require 'fileutils'
 require 'net/http'
 require 'json'
 require 'uri'
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
 
 describe Monotable::EventMachineServer do
   include DaemonTestHelper

spec/chunk_replication_spec.rb → spec/server_spec/chunk_replication_spec.rb

@@ -1,12 +1,11 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 require 'rubygems'
 require 'rest_client'
 require 'tmpdir'
 require 'fileutils'
 require 'net/http'
 require 'json'
 require 'uri'
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
 
 describe Monotable::EventMachineServer do
   include DaemonTestHelper
@@ -29,7 +28,7 @@
   it "should be possible replicate" do
     chunk_name = server_client.chunks[-1]
     server_client(0).set_chunk_replication_clients(chunk_name,[daemon_address(1)])
-    server_client(0).chunk_info(chunk_name)["replication_clients"].should == [daemon_address(1)]
+    server_client(0).chunk_status(chunk_name)["replication_clients"].should == [daemon_address(1)]
 
     server_client(1).chunks.should==[]
     server_client(1).clone_chunk(chunk_name,daemon_address(0))

spec/cluster_manager_spec.rb → spec/server_spec/cluster_manager_spec.rb

@@ -1,12 +1,11 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 require 'rubygems'
 require 'rest_client'
 require 'tmpdir'
 require 'fileutils'
 require 'net/http'
 require 'json'
 require 'uri'
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
 
 describe Monotable::ClusterManager do
   include DaemonTestHelper

spec/daemon_spec.rb → spec/server_spec/daemon_spec.rb

@@ -1,13 +1,11 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 require 'rubygems'
 require 'rest_client'
 require 'tmpdir'
 require 'fileutils'
 require 'net/http'
 require 'json'
 require 'uri'
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
-require File.expand_path(File.join(File.dirname(__FILE__),'mono_table_helper_methods'))
 
 describe Monotable::EventMachineServer do
   include DaemonTestHelper

spec/external_request_router_spec.rb → spec/server_spec/external_request_router_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::RequestRouter do
   include MonotableHelperMethods

spec/global_index_cache.rb → spec/server_spec/global_index_cache.rb

@@ -1,12 +1,11 @@
-require File.expand_path(File.join(File.dirname(__FILE__),'..','lib','monotable','monotable'))
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 require 'rubygems'
 require 'rest_client'
 require 'tmpdir'
 require 'fileutils'
 require 'net/http'
 require 'json'
 require 'uri'
-require File.expand_path(File.join(File.dirname(__FILE__),'daemon_test_helper'))
 
 describe Monotable::EventMachineServer do
   include DaemonTestHelper

spec/internal_request_router_spec.rb → spec/server_spec/internal_request_router_spec.rb

@@ -1,5 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
-require File.join(File.dirname(__FILE__),"common_api_tests")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::RequestRouter do
   include MonotableHelperMethods

spec/multi_daemon_spec.rb → spec/server_spec/multi_daemon_spec.rb

@@ -1,4 +1,4 @@
-require File.join(File.dirname(__FILE__),"mono_table_helper_methods")
+require File.join(File.dirname(__FILE__),"..","mono_table_helper_methods")
 
 describe Monotable::LocalStore do
   include MonotableHelperMethods