Permalink
Browse files

Add S3 multi-object delete operation

Conflicts:

	lib/awsbase/right_awsbase.rb
	lib/s3/right_s3_interface.rb
  • Loading branch information...
1 parent f128c3c commit bfe3e461d41bafd6a4758c6abee9ef89e5c01fdc @aboisvert aboisvert committed with konstantin-dzreev Dec 17, 2011
Showing with 104 additions and 3 deletions.
  1. +6 −1 lib/awsbase/right_awsbase.rb
  2. +49 −2 lib/s3/right_s3_interface.rb
  3. +49 −0 test/s3/test_right_s3.rb
View
7 lib/awsbase/right_awsbase.rb
@@ -43,7 +43,12 @@ def self.sign(aws_secret_access_key, auth_string)
Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
end
- # Escape a string accordingly Amazon rules
+ # Calculates 'Content-MD5' header value for some content
+ def self.content_md5(content)
+ Base64.encode64(Digest::MD5::new.update(content).digest).strip
+ end
+
+ # Escape a string accordingly Amazon rulles
# http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
def self.amz_escape(param)
param = param.flatten.join('') if param.is_a?(Array) # ruby 1.9.x Array#to_s fix
View
51 lib/s3/right_s3_interface.rb
@@ -52,9 +52,11 @@ class S3Interface < RightAwsBase
'response-content-encoding',
'torrent',
'uploadId',
- 'uploads'].sort
-
+ 'uploads',
+ 'delete'].sort
+ MULTI_OBJECT_DELETE_MAX_KEYS = 1000
+
@@bench = AwsBenchmarkingBlock.new
def self.bench_xml
@@bench.xml
@@ -788,6 +790,34 @@ def delete(bucket, key='', headers={})
on_exception
end
+ # Deletes multiple keys. Returns an array with errors, if any.
+ #
+ # s3.delete_multiple('my_awesome_bucket', ['key1', 'key2', ...)
+ # #=> [ { :key => 'key2', :code => 'AccessDenied', :message => "Access Denied" } ]
+ #
+ def delete_multiple(bucket, keys=[], headers={})
+ errors = []
+ keys = Array.new(keys)
+ while keys.length > 0
+ data = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ data += "<Delete>\n<Quiet>true</Quiet>\n"
+ keys.take(MULTI_OBJECT_DELETE_MAX_KEYS).each do |key|
+ data += "<Object><Key>#{AwsUtils::xml_escape(key)}</Key></Object>\n"
+ end
+ data += "</Delete>"
+ req_hash = generate_rest_request('POST', headers.merge(
+ :url => "#{bucket}?delete",
+ :data => data,
+ 'content-md5' => AwsUtils::content_md5(data)
+ ))
+ errors += request_info(req_hash, S3DeleteMultipleParser.new)
+ keys = keys.drop(MULTI_OBJECT_DELETE_MAX_KEYS)
+ end
+ errors
+ rescue
+ on_exception
+ end
+
# Copy an object.
# directive: :copy - copy meta-headers from source (default value)
# :replace - replace meta-headers by passed ones
@@ -1154,6 +1184,23 @@ def put_bucket_acl_link(bucket, acl_xml_doc, headers={})
on_exception
end
+ class S3DeleteMultipleParser < RightAWSParser # :nodoc:
+ def reset
+ @result = []
+ end
+ def tagstart(name, attributes)
+ @error = {}
+ end
+ def tagend(name)
+ case name
+ when 'Key' then current[:key] = @text
+ when 'Code' then current[:code] = @text
+ when 'Message' then current[:message] = @text
+ when 'Error' then @result << @error
+ end
+ end
+ end
+
#-----------------------------------------------------------------
# PARSERS:
#-----------------------------------------------------------------
View
49 test/s3/test_right_s3.rb
@@ -482,6 +482,55 @@ def test_43_add_expires_and_content_type_and_content_disposition
end
end
+ def test_44_delete_multiple
+ bucket = RightAws::S3::Bucket.create(@s, @bucket, true)
+
+ key1 = Rightscale::S3::Key.create(bucket, @key1)
+ key2 = Rightscale::S3::Key.create(bucket, @key2)
+ key3 = Rightscale::S3::Key.create(bucket, @key3)
+
+ assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+
+ key1.refresh
+ key2.refresh
+ key3.refresh
+
+ assert key1.exists?
+ assert key2.exists?
+ assert key3.exists?
+
+ result = @s3.delete_multiple(@bucket, [@key1, @key2, @key3])
+ assert result.empty?
+
+ key1.refresh
+ key2.refresh
+ key3.refresh
+
+ assert !key1.exists?
+ assert !key2.exists?
+ assert !key3.exists?
+ end
+
+ def test_45_delete_multiple_more_than_1000_objects
+ n = 1200
+ keys = (1..n).map { |i| "key-#{i}"}
+
+ keys.each do |key|
+ assert @s3.put(@bucket, key, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ end
+
+ result = @s3.delete_multiple(@bucket, keys)
+ assert result.empty?
+
+ keys_after = @s3.list_bucket(@bucket).map { |obj| obj[:key] }
+
+ keys.each do |key|
+ keys_after.include?(key)
+ end
+ end
+
private
def request( uri )

0 comments on commit bfe3e46

Please sign in to comment.