Merge pull request #27 from zendesk/alex/custom-slice-size

custom slice size

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    large_object_store (1.6.1)
+    large_object_store (1.7.0)
       zstd-ruby (~> 1.5.5)
 
 GEM

lib/large_object_store.rb

@@ -27,21 +27,18 @@ class << self
   class RailsWrapper
     attr_reader :store
 
-    def initialize(store, serializer: Marshal)
+    def initialize(store, serializer: Marshal, max_slice_size: MAX_OBJECT_SIZE)
       @store = store
       @serializer = serializer
+      @max_slice_size = [max_slice_size, MAX_OBJECT_SIZE].min
+      @namespace = (store.respond_to?(:options) && store.options[:namespace]) || ""
     end
 
     def write(key, value, **options)
       options = options.dup
       value = serialize(value, options)
 
-      # calculate slice size; note that key length is a factor because
-      # the key is stored on the same slab page as the value
-      namespace = (store.respond_to?(:options) && store.options[:namespace]) || ""
-      namespace_length = namespace.empty? ? 0 : namespace.size + 1
-      slice_size = MAX_OBJECT_SIZE - ITEM_HEADER_SIZE - UUID_SIZE - key.bytesize - namespace_length
-
+      slice_size = safe_slice_size(key)
       # store number of pages
       pages = (value.size / slice_size.to_f).ceil
 
@@ -110,6 +107,19 @@ def delete(key)
 
     private
 
+    # calculate slice size; note that key length is a factor because
+    # the key is stored on the same slab page as the value
+    def safe_slice_size(key)
+      namespace_length = @namespace.empty? ? 0 : @namespace.size + 1
+      overhead = ITEM_HEADER_SIZE + UUID_SIZE + key.bytesize + namespace_length
+      slice_size = @max_slice_size - overhead
+      if slice_size <= 0
+        MAX_OBJECT_SIZE - overhead
+      else
+        slice_size
+      end
+    end
+
     # convert a object to a string
     # modifies options
     def serialize(value, options)

lib/large_object_store/version.rb

@@ -1,3 +1,3 @@
 module LargeObjectStore
-  VERSION = "1.6.1"
+  VERSION = "1.7.0"
 end

spec/large_object_store_spec.rb

@@ -22,6 +22,12 @@ def type(data, kind)
     end
   end
 
+  def with_namespace(store, namespace)
+    store.instance_variable_set('@namespace', namespace)
+    yield
+    store.instance_variable_set('@namespace', '')
+  end
+
   stores = [ActiveSupport::Cache::MemoryStore.new]
 
   begin
@@ -35,6 +41,7 @@ def type(data, kind)
     describe "with #{cache_instance.class} as the base store" do
       let(:cache) { cache_instance }
       let(:store) { LargeObjectStore.wrap(cache) }
+      let(:custom_slice_store) { LargeObjectStore.wrap(cache, max_slice_size: 100_000) }
       let(:version) { LargeObjectStore::CACHE_VERSION }
 
       before { cache.clear }
@@ -221,6 +228,26 @@ def type(data, kind)
         expect(store.store.read("#{key}_#{version}_1").size).to eq(1048576 - 100 - 250)
       end
 
+      it "adjusts slice size for namespace length" do
+        with_namespace(store, 'los') do
+          expect(store.write("a", "a"*20_000_000)).to eq(true)
+          expect(store.store.read("a_#{version}_1").size).to eq(1048576 - 100 - 4 - 1)
+
+          key="a"*250
+          expect(store.write(key, "a"*20_000_000)).to eq(true)
+          expect(store.store.read("#{key}_#{version}_1").size).to eq(1048576 - 100 - 250 - 4)
+        end
+      end
+
+      it "adjusts slice size for custom max_slice_size" do
+        expect(custom_slice_store.write("a", "a"*20_000_000)).to eq(true)
+        expect(custom_slice_store.store.read("a_#{version}_1").size).to eq(100_000 - 100 - 1)
+
+        key="a"*250
+        expect(custom_slice_store.write(key, "a"*20_000_000)).to eq(true)
+        expect(custom_slice_store.store.read("#{key}_#{version}_1").size).to eq(100_000 - 100 - 250)
+      end
+
       it "uses necessary keys" do
         store.write("a", "a"*5_000_000)
         expect(["a_#{version}_0", "a_#{version}_1", "a_#{version}_2", "a_#{version}_3", "a_#{version}_4", "a_#{version}_5", "a_#{version}_6"].map do |k|