Merge pull request #1636 from nanoc/denis/reliable-bin-write

Let Store#store support large amounts of data

nanoc-core/lib/nanoc/core/store.rb

@@ -128,8 +128,9 @@ def unsafe_load_uninstrumented
         self.data = read_obj_from_file(data_filename)
       end
 
-      def write_obj_to_file(fn, obj)
-        File.binwrite(fn, Marshal.dump(obj))
+      def write_obj_to_file(filename, obj)
+        data = Marshal.dump(obj)
+        write_data_to_file(filename, data)
       end
 
       def read_obj_from_file(fn)
@@ -143,6 +144,28 @@ def version_filename
       def data_filename
         "#{filename}.data.db"
       end
+
+      def write_data_to_file(filename, data)
+        basename = File.basename(filename)
+        dirname = File.dirname(filename)
+
+        # Write to a temporary file first, and then (atomically) move it into
+        # place.
+        Tempfile.open(".#{basename}", dirname) do |temp_file|
+          temp_file.binmode
+
+          # Write the data as a stream, because File.binwrite can’t
+          # necessarily deal with writing that much data all at once.
+          #
+          # See https://github.com/nanoc/nanoc/issues/1635.
+          reader = StringIO.new(data)
+          IO.copy_stream(reader, temp_file)
+          temp_file.close
+
+          # Rename (atomic)
+          File.rename(temp_file.path, filename)
+        end
+      end
     end
   end
 end

nanoc-core/spec/nanoc/core/store_spec.rb

@@ -109,4 +109,25 @@ def gen_hash(path)
     store.load
     expect(store.data).to be_nil
   end
+
+  it 'can write humongous amounts of data' do
+    # Skip running on GitHub actions etc because this thing just uses far too many resources
+    skip if ENV['CI']
+
+    store = test_store_klass.new('test', 1)
+
+    # Create huge string
+    array = []
+    100.times do |i|
+      raw = 'x' * 1_000_037
+      raw << i.to_s
+      io = StringIO.new
+      100.times { io << raw }
+      array << io.string
+    end
+
+    # Write
+    store.data = { data: array }
+    expect { store.store }.not_to raise_exception
+  end
 end