Skip to content

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also .

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also .
...
  • 5 commits
  • 4 files changed
  • 0 commit comments
  • 2 contributors
Showing with 109 additions and 7 deletions.
  1. +1 −0 History.txt
  2. +6 −1 lib/awsbase/right_awsbase.rb
  3. +49 −2 lib/s3/right_s3_interface.rb
  4. +53 −4 test/s3/test_right_s3.rb
View
1 History.txt
@@ -351,3 +351,4 @@ the source key.
Release Notes:
- Fixed:
- Single-threaded multipart upload support (https://github.com/rightscale/right_aws/pull/116)
+ - S3 multi object delete (https://github.com/rightscale/right_aws/pull/106)
View
7 lib/awsbase/right_awsbase.rb
@@ -43,7 +43,12 @@ def self.sign(aws_secret_access_key, auth_string)
Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
end
- # Escape a string accordingly Amazon rules
+ # Calculates 'Content-MD5' header value for some content
+ def self.content_md5(content)
+ Base64.encode64(Digest::MD5::new.update(content).digest).strip
+ end
+
+ # Escape a string accordingly Amazon rulles
# http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
def self.amz_escape(param)
param = param.flatten.join('') if param.is_a?(Array) # ruby 1.9.x Array#to_s fix
View
51 lib/s3/right_s3_interface.rb
@@ -52,9 +52,11 @@ class S3Interface < RightAwsBase
'response-content-encoding',
'torrent',
'uploadId',
- 'uploads'].sort
-
+ 'uploads',
+ 'delete'].sort
+ MULTI_OBJECT_DELETE_MAX_KEYS = 1000
+
@@bench = AwsBenchmarkingBlock.new
def self.bench_xml
@@bench.xml
@@ -788,6 +790,34 @@ def delete(bucket, key='', headers={})
on_exception
end
+ # Deletes multiple keys. Returns an array with errors, if any.
+ #
+ # s3.delete_multiple('my_awesome_bucket', ['key1', 'key2', ...)
+ # #=> [ { :key => 'key2', :code => 'AccessDenied', :message => "Access Denied" } ]
+ #
+ def delete_multiple(bucket, keys=[], headers={})
+ errors = []
+ keys = Array.new(keys)
+ while keys.length > 0
+ data = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n"
+ data += "<Delete>\n<Quiet>true</Quiet>\n"
+ keys.take(MULTI_OBJECT_DELETE_MAX_KEYS).each do |key|
+ data += "<Object><Key>#{AwsUtils::xml_escape(key)}</Key></Object>\n"
+ end
+ data += "</Delete>"
+ req_hash = generate_rest_request('POST', headers.merge(
+ :url => "#{bucket}?delete",
+ :data => data,
+ 'content-md5' => AwsUtils::content_md5(data)
+ ))
+ errors += request_info(req_hash, S3DeleteMultipleParser.new)
+ keys = keys.drop(MULTI_OBJECT_DELETE_MAX_KEYS)
+ end
+ errors
+ rescue
+ on_exception
+ end
+
# Copy an object.
# directive: :copy - copy meta-headers from source (default value)
# :replace - replace meta-headers by passed ones
@@ -1154,6 +1184,23 @@ def put_bucket_acl_link(bucket, acl_xml_doc, headers={})
on_exception
end
+ class S3DeleteMultipleParser < RightAWSParser # :nodoc:
+ def reset
+ @result = []
+ end
+ def tagstart(name, attributes)
+ @error = {} if name == 'Error'
+ end
+ def tagend(name)
+ case name
+ when 'Key' then @error[:key] = @text
+ when 'Code' then @error[:code] = @text
+ when 'Message' then @error[:message] = @text
+ when 'Error' then @result << @error
+ end
+ end
+ end
+
#-----------------------------------------------------------------
# PARSERS:
#-----------------------------------------------------------------
View
57 test/s3/test_right_s3.rb
@@ -433,7 +433,7 @@ def test_37_access_logging
sleep 10
assert_equal({:enabled => true, :targetbucket => @bucket2, :targetprefix => "loggylogs/"}, bucket.logging_info)
-
+
assert bucket.disable_logging
# check 'Drop' method
@@ -482,14 +482,63 @@ def test_43_add_expires_and_content_type_and_content_disposition
end
end
+ def test_44_delete_multiple
+ bucket = RightAws::S3::Bucket.create(@s, @bucket, true)
+
+ key1 = Rightscale::S3::Key.create(bucket, @key1)
+ key2 = Rightscale::S3::Key.create(bucket, @key2)
+ key3 = Rightscale::S3::Key.create(bucket, @key3)
+
+ assert @s3.put(@bucket, @key1, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ assert @s3.put(@bucket, @key2, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ assert @s3.put(@bucket, @key3, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+
+ key1.refresh
+ key2.refresh
+ key3.refresh
+
+ assert key1.exists?
+ assert key2.exists?
+ assert key3.exists?
+
+ result = @s3.delete_multiple(@bucket, [@key1, @key2, @key3])
+ assert result.empty?
+
+ key1.refresh
+ key2.refresh
+ key3.refresh
+
+ assert !key1.exists?
+ assert !key2.exists?
+ assert !key3.exists?
+ end
+
+ def test_45_delete_multiple_more_than_1000_objects
+ n = 1200
+ keys = (1..n).map { |i| "key-#{i}"}
+
+ keys.each do |key|
+ assert @s3.put(@bucket, key, RIGHT_OBJECT_TEXT), 'Put bucket fail'
+ end
+
+ result = @s3.delete_multiple(@bucket, keys)
+ assert result.empty?
+
+ keys_after = @s3.list_bucket(@bucket).map { |obj| obj[:key] }
+
+ keys.each do |key|
+ assert !keys_after.include?(key)
+ end
+ end
+
private
def request( uri )
url = URI.parse( uri )
- http = Net::HTTP.new(url.host, 443)
- http.use_ssl = true
- http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+ http = Net::HTTP.new(url.host, 80)
+# http.use_ssl = true
+# http.verify_mode = OpenSSL::SSL::VERIFY_PEER
http.request(Net::HTTP::Get.new( url.request_uri ))
end

No commit comments for this range

Something went wrong with that request. Please try again.