From 808f8bbdca6d5c6eca1f9addeac45d762111b160 Mon Sep 17 00:00:00 2001 From: alor Date: Tue, 5 Feb 2013 16:59:37 +0100 Subject: [PATCH 1/2] added support for append mode in gridfs --- lib/mongo/gridfs/grid_io.rb | 61 ++++++++++++++++++++++++++++++--- test/functional/grid_io_test.rb | 37 ++++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/lib/mongo/gridfs/grid_io.rb b/lib/mongo/gridfs/grid_io.rb index ac7c726d4f..7cef8ece91 100644 --- a/lib/mongo/gridfs/grid_io.rb +++ b/lib/mongo/gridfs/grid_io.rb @@ -53,8 +53,9 @@ def initialize(files, chunks, filename, mode, opts={}) case @mode when 'r' then init_read when 'w' then init_write(opts) + when 'a' then init_append(opts) else - raise GridError, "Invalid file mode #{@mode}. Mode should be 'r' or 'w'." + raise GridError, "Invalid file mode #{@mode}. Mode should be 'r', 'w' or 'a'." end end @@ -98,7 +99,8 @@ def read(length=nil) # @return [Integer] # the number of bytes written. def write(io) - raise GridError, "file not opened for write" unless @mode[0] == ?w + raise GridError, "file not opened for write" unless @mode[0] == ?w or @mode[0] == ?a + if io.is_a? String if Mongo::WriteConcern.gle?(@write_concern) @local_md5.update(io) @@ -216,13 +218,27 @@ def getc # on GridIO#open is passed a block. Otherwise, it must be called manually. # # @return [BSON::ObjectId] + #def close + # if @current_chunk['n'].zero? && @chunk_position.zero? + # warn "Warning: Storing a file with zero length." + # end + # @upload_date = Time.now.utc + # object = to_mongo_object + # id = @files.update({_id: object['_id']}, object, {upsert: true}) + # id + #end def close - if @mode[0] == ?w + if @mode[0] == ?w or @mode[0] == ?a if @current_chunk['n'].zero? && @chunk_position.zero? warn "Warning: Storing a file with zero length." end @upload_date = Time.now.utc - id = @files.insert(to_mongo_object) + if @mode[0] == ?w + id = @files.insert(to_mongo_object) + elsif @mode[0] == ?a + object = to_mongo_object + id = @files.update({_id: object['_id']}, object, {upsert: true}) + end end id end @@ -267,7 +283,7 @@ def save_chunk(chunk) def get_chunk(n) chunk = @chunks.find({'files_id' => @files_id, 'n' => n}).next_document - @chunk_position = 0 + @chunk_position = (@mode == ?a ? chunk['data'].size : 0) unless chunk.nil? chunk end @@ -420,7 +436,41 @@ def init_write(opts) @current_chunk = create_chunk(0) @file_position = 0 end + + # Initialize the class for appending to a file. + def init_append(opts) + doc = @files.find(@query, @query_opts).next_document + return init_write(opts) unless doc + + opts = doc.dup + + @files_id = opts.delete('_id') + @content_type = opts.delete('contentType') + @chunk_size = opts.delete('chunkSize') + @upload_date = opts.delete('uploadDate') + @aliases = opts.delete('aliases') + @file_length = opts.delete('length') + @metadata = opts.delete('metadata') + @md5 = opts.delete('md5') + @filename = opts.delete('filename') + @custom_attrs = opts + + # recalculate the md5 of the previous chunks + if Mongo::WriteConcern.gle?(@write_concern) + @current_chunk = get_chunk(0) + @file_position = 0 + @local_md5 = Digest::MD5.new + @local_md5.update(read_all) + end + # position at the end of the file (last chunk) + last_chunk = @file_length / @chunk_size + @current_chunk = get_chunk(last_chunk) + chunk = get_chunk(last_chunk-1) if @current_chunk.nil? + @current_chunk ||= create_chunk(last_chunk) + @file_position = @chunk_size * last_chunk + @current_chunk['data'].size + end + def check_existing_file if @files.find_one('_id' => @files_id) raise GridError, "Attempting to overwrite with Grid#put. You must delete the file first." @@ -448,6 +498,7 @@ def get_md5 md5_command['filemd5'] = @files_id md5_command['root'] = @fs_name @server_md5 = @files.db.command(md5_command)['md5'] + if Mongo::WriteConcern.gle?(@write_concern) @client_md5 = @local_md5.hexdigest if @local_md5 == @server_md5 diff --git a/test/functional/grid_io_test.rb b/test/functional/grid_io_test.rb index a59077306f..18dd44b202 100644 --- a/test/functional/grid_io_test.rb +++ b/test/functional/grid_io_test.rb @@ -151,6 +151,43 @@ class GridIOTest < Test::Unit::TestCase end end + context "Appending" do + setup do + @filename = 'test' + @length = nil + @times = 2 + end + + should "correctly append two chunks" do + + @times.times do |t| + file = GridIO.new(@files, @chunks, @filename, 'a') + file.write "#{t}" * 100 + file.close + end + + file = GridIO.new(@files, @chunks, @filename, 'r') + data = file.read + file.close + + assert_equal data, @times.times.map {|i| "#{i}" * 100}.join + end + + should "append two chunks of exactly chunk_size" do + @times.times do |t| + file = GridIO.new(@files, @chunks, @filename, 'a') + file.write "#{t}" * file.chunk_size + file.close + end + + file = GridIO.new(@files, @chunks, @filename, 'r') + data = file.read + file.close + + assert_equal data, @times.times.map {|i| "#{i}" * file.chunk_size}.join + end + end + context "Seeking" do setup do @filename = 'test' From ed2c6e040abeeb4d682bb64aaea266749a9979f5 Mon Sep 17 00:00:00 2001 From: alor Date: Wed, 6 Feb 2013 14:41:32 +0100 Subject: [PATCH 2/2] backport to ruby 1.8.x --- lib/mongo/gridfs/grid_io.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongo/gridfs/grid_io.rb b/lib/mongo/gridfs/grid_io.rb index 7cef8ece91..093b36df4b 100644 --- a/lib/mongo/gridfs/grid_io.rb +++ b/lib/mongo/gridfs/grid_io.rb @@ -237,7 +237,7 @@ def close id = @files.insert(to_mongo_object) elsif @mode[0] == ?a object = to_mongo_object - id = @files.update({_id: object['_id']}, object, {upsert: true}) + id = @files.update({:_id => object['_id']}, object, {:upsert => true}) end end id