Skip to content

Commit

Permalink
Let Store#store support large amounts of data
Browse files Browse the repository at this point in the history
  • Loading branch information
denisdefreyne committed Dec 1, 2022
1 parent 2d05b82 commit b5f5c0a
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 2 deletions.
24 changes: 22 additions & 2 deletions nanoc-core/lib/nanoc/core/store.rb
Expand Up @@ -128,8 +128,28 @@ def unsafe_load_uninstrumented
self.data = read_obj_from_file(data_filename)
end

def write_obj_to_file(fn, obj)
File.binwrite(fn, Marshal.dump(obj))
def write_obj_to_file(filename, obj)
chunk_size = 104_857_600 # 100 MiB

data = Marshal.dump(obj)

# Read the marshalled data in chunks, because File.binwrite can’t
# necessarily deal with writing that much data all at once.
#
# See https://github.com/nanoc/nanoc/issues/1635.
reader = StringIO.new(data)
File.open(filename, 'wb:ASCII-8BIT') do |writer|
# Create a string to read into, to reduce the amount of allocation.
#
# Note that String.new takes a `capacity` kwarg, but this isn’t used
# here as the performance impact seemed to be negligible or even
# slightly negative.
chunk = +''

while reader.read(chunk_size, chunk) # rubocop:disable Style/WhileUntilModifier
writer.write(chunk)
end
end
end

def read_obj_from_file(fn)
Expand Down
21 changes: 21 additions & 0 deletions nanoc-core/spec/nanoc/core/store_spec.rb
Expand Up @@ -109,4 +109,25 @@ def gen_hash(path)
store.load
expect(store.data).to be_nil
end

it 'can write humongous amounts of data' do
# Skip running on GitHub actions etc because this thing just uses far too many resources
skip if ENV['CI']

store = test_store_klass.new('test', 1)

# Create huge string
array = []
100.times do |i|
raw = 'x' * 1_000_037
raw << i.to_s
io = StringIO.new
100.times { io << raw }
array << io.string
end

# Write
store.data = { data: array }
expect { store.store }.not_to raise_exception
end
end

0 comments on commit b5f5c0a

Please sign in to comment.