Permalink
Browse files

Adds 'list_parts' and a test for s3 multipart uploads

  • Loading branch information...
marios committed Oct 30, 2012
1 parent 2c3ae27 commit b11283f2f18a9e37e581df4fd3b0fdc4e7caea4a
Showing with 81 additions and 9 deletions.
  1. +62 −8 lib/s3/s3_interface.rb
  2. +19 −1 test/s3/test_s3.rb
View
@@ -346,7 +346,7 @@ def incrementally_list_bucket(bucket, options={}, headers={}, &block)
begin
internal_bucket = bucket.dup
unless internal_options.nil? || internal_options.empty?
- internal_bucket << '?'
+ internal_bucket << '?'
internal_bucket << internal_options.map { |k, v| "#{k.to_s}=#{CGI::escape v.to_s}" }.join('&')
end
req_hash = generate_rest_request('GET', headers.merge(:url=>internal_bucket))
@@ -547,7 +547,7 @@ def initiate_multipart(bucket, key, headers={})
# Among the parameters required, clients must supply the uploadId (obtained from the initiate_multipart method call) as
# well as the partNumber for this part (user-specified, determining the sequence for reassembly).
#
- # s3.upload_part('my_awesome_bucket', 'hugeObject', "WL7dk8sqbtk3Rg641HHWaNeG6RxI", "2", File.open('localfilename.dat'))
+ # s3.upload_part('my_awesome_bucket', 'hugeObject', "WL7dk8sqbtk3Rg641HHWaNeG6RxI", "2", File.open('localfilename.dat'))
# => "b54357faf0632cce46e942fa68356b38"
#
# The return Etag must be retained for use in the completion of the multipart upload; see
@@ -580,9 +580,9 @@ def upload_part(bucket, key, uploadId, partNumber, data, headers={})
# Clients must specify the uploadId (obtained from the initiate_multipart call) and the reassembly manifest hash
# which specifies the each partNumber corresponding etag (obtained from the upload_part call):
#
- # s3.complete_multipart('my_awesome_bucket', 'hugeObject', "WL7dk8sqbtk3Rg641HHWaNeG6RxI",
+ # s3.complete_multipart('my_awesome_bucket', 'hugeObject', "WL7dk8sqbtk3Rg641HHWaNeG6RxI",
# {"1"=>"a54357aff0632cce46d942af68356b38", "2"=>"0c78aef83f66abc1fa1e8477f296d394"}) => true
- #
+ #
# See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html
#
def complete_multipart(bucket, key, uploadId, manifest_hash, headers={})
@@ -607,6 +607,31 @@ def complete_multipart(bucket, key, uploadId, manifest_hash, headers={})
on_exception
end
+ # List parts of a multipart upload, returning a hash or an exception
+ # http://docs.amazonwebservices.com/AmazonS3/latest/API/mpUploadListParts.html
+ #
+ # response looks like:
+ #
+ # { :bucket=>"mariosFoo_awesome_test_bucket_000A1",
+ # :key=>"mariosFoosegmented",
+ # :upload_id=>"jQKX7JdJBTrbvLn9apUPIXkt14FHdp6nMZVg--"
+ # :parts=> [ {:part_number=>"1", :last_modified=>"2012-10-30T15:06:28.000Z",
+ # :etag=>"\"78f871f6f01673a4aca05b1f8e26df08\"", :size=>"6276589"},
+ # {:part_number=>"2", :last_modified=>"2012-10-30T15:08:22.000Z",
+ # :etag=>"\"e7b94a1e959ca066026da3ec63aad321\"", :size=>"7454095"}] }
+ #
+ # Clients must specify the uploadId (obtained from the initiate_multipart call)
+ #
+ # s3.list_parts('my_awesome_bucket', 'hugeObject', "WL7dk8sqbtk3Rg641HHWaNeG6RxI",
+ #
+ # See http://docs.amazonwebservices.com/AmazonS3/latest/dev/mpuoverview.html
+ #
+ def list_parts(bucket, key, uploadId, headers={})
+ req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}?uploadId=#{uploadId}"))
+ request_info(req_hash, S3ListMultipartPartsParser.new)
+ end
+
+
# Retrieves object data from Amazon. Returns a +hash+ or an exception.
#
# s3.get('my_awesome_bucket', 'log/curent/1.log') #=>
@@ -633,7 +658,6 @@ def complete_multipart(bucket, key, uploadId, manifest_hash, headers={})
# end
# foo.close
#
-
def get(bucket, key, headers={}, &block)
req_hash = generate_rest_request('GET', headers.merge(:url=>"#{bucket}/#{CGI::escape key}"))
request_info(req_hash, S3HttpResponseBodyParser.new, &block)
@@ -1335,18 +1359,42 @@ def tagend(name)
end
- class S3UploadPartParser < AwsParser # :nodoc:
+ class S3ListMultipartPartsParser < AwsParser # :nodoc:
def reset
- @result = ""
+ @result = {}
+ @result[:parts] = []
+ @current_part={}
+ end
+
+ def tagstart(name, attributes)
+ @current_part={} if name=='Part'
end
def tagend(name)
- @result = @text if name == 'ETag'
+ case name
+ when 'PartNumber'
+ @current_part[:part_number] = @text
+ when 'ETag'
+ @current_part[:etag] = @text
+ when 'Size'
+ @current_part[:size] = @text
+ when 'LastModified'
+ @current_part[:last_modified] = @text
+ when 'Bucket'
+ @result[:bucket] = @text
+ when 'Key'
+ @result[:key] = @text
+ when 'UploadId'
+ @result[:upload_id] = @text
+ when 'Part'
+ @result[:parts] << @current_part
+ end
end
end
+
#-----------------------------------------------------------------
# PARSERS: Non XML
#-----------------------------------------------------------------
@@ -1386,6 +1434,12 @@ def parse(response)
end
end
+ class S3UploadPartParser < S3HttpResponseParser # :nodoc:
+ def parse(response)
+ @result = headers_to_string(response.to_hash)["etag"].gsub!("\"", "")
+ end
+ end
+
end
end
View
@@ -126,9 +126,27 @@ def test_13_delete_folder
assert_equal 1, @s3.delete_folder(@bucket, 'test').size, "Only one key(#{@key1}) must be deleted!"
end
+ def test_14_multipart_upload
+ segmented_object = TestCredentials.config['amazon']['my_prefix']+"segmented"
+ uploadId = @s3.initiate_multipart(@bucket, segmented_object)
+ assert(uploadId.instance_of?(String))
+ part1_etag = @s3.upload_part(@bucket, segmented_object, uploadId, "1", File.open(TestCredentials.config['amazon']['multipart_segment1']))
+ assert(part1_etag.instance_of?(String))
+ part2_etag = @s3.upload_part(@bucket, segmented_object, uploadId, "2", File.open(TestCredentials.config['amazon']['multipart_segment2']))
+ assert(part2_etag.instance_of?(String))
+ parts = @s3.list_parts(@bucket, segmented_object, uploadId)
+ part_etags = parts[:parts].collect{|part| part[:etag].gsub!("\"", "")}
+ assert(part_etags.include?(part1_etag))
+ assert(part_etags.include?(part2_etag))
+ assert(@s3.complete_multipart(@bucket, segmented_object, uploadId, {"1"=>part1_etag, "2"=>part2_etag}))
+ object_data = @s3.head(@bucket, segmented_object)
+ combined_size = File.size(TestCredentials.config['amazon']['multipart_segment1'])+ File.size(TestCredentials.config['amazon']['multipart_segment2'])
+ assert_equal object_data["content-length"].to_i, combined_size
+ end
+
# idle timeout is 20 seconds
# https://forums.aws.amazon.com/thread.jspa?threadID=58038
- def test_14_idle_timeout
+ def test_15_idle_timeout
@s3 = Aws::S3Interface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key,
:connection_mode=>:single)
# Disable connection retrying

0 comments on commit b11283f

Please sign in to comment.