Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Version 1.6.3 of the AWS SDK for Ruby

  • Loading branch information...
commit 9877f13179fdffaa31484416ecf690d52597a767 1 parent b91a8ba
@trevorrowe trevorrowe authored
Showing with 3,661 additions and 883 deletions.
  1. +1 −0  Gemfile
  2. +2 −1  features/ec2/step_definitions/ec2.rb
  3. +9 −1 features/ec2/step_definitions/snapshot_attributes.rb
  4. +326 −0 features/s3/high_level/client_side_encryption.feature
  5. +0 −5 features/s3/high_level/objects.feature
  6. +235 −0 features/s3/high_level/step_definitions/client_side_encryption.rb
  7. +1 −1  features/s3/high_level/step_definitions/objects.rb
  8. +6 −2 features/s3/high_level/step_definitions/versions.rb
  9. +2 −2 features/s3/high_level/versions.feature
  10. +6 −2 features/support/common.rb
  11. +13 −2 lib/aws/core.rb
  12. +1 −1  lib/aws/core/autoloader.rb
  13. +69 −30 lib/aws/core/client.rb
  14. +12 −1 lib/aws/core/configuration.rb
  15. +28 −16 lib/aws/core/http/handler.rb
  16. +31 −11 lib/aws/core/http/net_http_handler.rb
  17. +52 −16 lib/aws/core/http/request.rb
  18. +20 −16 lib/aws/core/http/response.rb
  19. +14 −14 lib/aws/core/indifferent_hash.rb
  20. +1 −0  lib/aws/core/query_client.rb
  21. +32 −14 lib/aws/core/response.rb
  22. +1 −0  lib/aws/core/signature/version_2.rb
  23. +16 −16 lib/aws/core/signature/version_4.rb
  24. +2 −2 lib/aws/dynamo_db/client.rb
  25. +0 −6 lib/aws/dynamo_db/request.rb
  26. +10 −10 lib/aws/rails.rb
  27. +44 −29 lib/aws/s3.rb
  28. +171 −6 lib/aws/s3/bucket.rb
  29. +119 −0 lib/aws/s3/cipher_io.rb
  30. +75 −45 lib/aws/s3/client.rb
  31. +6 −0 lib/aws/s3/config.rb
  32. +136 −49 lib/aws/s3/data_options.rb
  33. +144 −0 lib/aws/s3/encryption_utils.rb
  34. +14 −0 lib/aws/s3/errors.rb
  35. +7 −4 lib/aws/s3/multipart_upload.rb
  36. +2 −2 lib/aws/s3/object_collection.rb
  37. +1 −1  lib/aws/s3/policy.rb
  38. +21 −33 lib/aws/s3/request.rb
  39. +797 −237 lib/aws/s3/s3_object.rb
  40. +0 −2  lib/aws/simple_email_service/request.rb
  41. +0 −3  lib/aws/simple_workflow/request.rb
  42. +63 −75 lib/net/http/connection_pool.rb
  43. +69 −15 lib/net/http/connection_pool/connection.rb
  44. +39 −6 lib/net/http/connection_pool/session.rb
  45. +1 −1  spec/aws/config_spec.rb
  46. +23 −10 spec/aws/core/client_spec.rb
  47. +2 −2 spec/aws/core/http/httparty_handler_spec.rb
  48. +23 −21 spec/aws/core/http/net_http_handler_spec.rb
  49. +45 −15 spec/aws/core/http/request_spec.rb
  50. +2 −2 spec/aws/core/http/response_spec.rb
  51. +16 −15 spec/aws/core/response_spec.rb
  52. +115 −0 spec/aws/s3/cipher_io_spec.rb
  53. +25 −75 spec/aws/s3/client_spec.rb
  54. +674 −0 spec/aws/s3/s3_object_encrypt_spec.rb
  55. +86 −35 spec/aws/s3/s3_object_spec.rb
  56. +18 −9 spec/net/http/connection_pool/session_spec.rb
  57. +30 −22 spec/shared/aws_client_examples.rb
  58. +3 −0  spec/spec_helper.rb
View
1  Gemfile
@@ -39,6 +39,7 @@ group :build do
gem 'rspec', '2.5'
gem 'rspec', '2.5', :require => 'rspec/core/rake_task'
gem 'rcov', '0.9.9'
+ gem 'simplecov', :require => false
gem 'ci_reporter', '~> 1.6', :require => 'ci/reporter/rake/rspec'
end
View
3  features/ec2/step_definitions/ec2.rb
@@ -131,7 +131,8 @@
@created_volumes.each do |volume|
begin
volume.delete
- rescue AWS::EC2::Errors::InvalidVolumeID::NotFound
+ rescue AWS::EC2::Errors::InvalidVolumeID::NotFound,
+ AWS::EC2::Errors::InvalidVolume::NotFound
# already deleted
end
end
View
10 features/ec2/step_definitions/snapshot_attributes.rb
@@ -41,9 +41,17 @@
end
Given /^I create a snapshot with description "([^\"]*)"$/ do |description|
+
Given %(I create a volume)
- @snapshot = @volume.create_snapshot(description)
+
+ # it can take a while before the volume is in a state where snapshots
+ # are allowed
+ eventually do
+ @snapshot = @volume.create_snapshot(description)
+ end
+
@created_snapshots << @snapshot
+
end
Then /^the snapshot description should be "([^\"]*)"$/ do |description|
View
326 features/s3/high_level/client_side_encryption.feature
@@ -0,0 +1,326 @@
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+# language: en
+@s3 @high_level @objects @cse
+Feature: Amazon S3 Client Side Encryption
+
+ As a customer of Amazon S3
+ I want my data to be envelope encrypted within Amazon S3
+ So that I can store my private data securely while managing my own key
+
+ @cse_asym @cse_write
+ Scenario: Write an object with CSE
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify no CSE material location
+ When I write the string "HELLO" to it using CSE
+ And I specify metadata CSE material location
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_asym @cse_read
+ Scenario: Read an object with CSE
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify no CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I specify metadata CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+
+ @cse_asym @cse_read
+ Scenario: Read an object with CSE with metadata specified
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I read it with the CSE key
+ And I specify metadata CSE material location
+ Then the result should be "HELLO"
+
+ @cse_asym @cse_write
+ Scenario: Write an object with CSE with instruction file material location
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify instruction file CSE material location
+ When I write the string "HELLO" to it using CSE
+ And I specify instruction file CSE material location
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with an instruction file
+ And the object should have unencrypted_content_length
+
+# @cse_asym @cse_read
+# Scenario: Read an object with CSE with instruction file material location when not specified
+# Given I ask for the object with key "foo"
+# And I have an asymmetric CSE key
+# And I specify instruction file CSE material location
+# And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+# When I read it with the CSE key
+# And I specify no CSE material location
+# Then the result should be "HELLO"
+
+ @cse_asym @cse_read
+ Scenario: Read an object with CSE with instruction file material location when specified
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify instruction file CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I read it with the CSE key
+ Then the result should be "HELLO"
+
+ @cse_sym @cse_write
+ Scenario: Write an object with CSE symmetric
+ Given I ask for the object with key "foo"
+ And I have a symmetric "128" CSE key
+ And I specify no CSE material location
+ When I write the string "HELLO" to it using CSE
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_sym @cse_read
+ Scenario: Read an object with CSE symmetric
+ Given I ask for the object with key "foo"
+ And I have a symmetric "192" CSE key
+ And I specify no CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I specify metadata CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+
+ @cse_sym @cse_read
+ Scenario: Read an object with CSE symmetric with metadata specified
+ Given I ask for the object with key "foo"
+ And I have a symmetric "256" CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I read it with the CSE key
+ And I specify metadata CSE material location
+ Then the result should be "HELLO"
+
+ @cse_sym @cse_write
+ Scenario: Write an object with CSE symmetric with instruction file material location
+ Given I ask for the object with key "foo"
+ And I have a symmetric "192" CSE key
+ And I specify instruction file CSE material location
+ When I write the string "HELLO" to it using CSE
+ And I specify instruction file CSE material location
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with an instruction file
+ And the object should have unencrypted_content_length
+
+ @cse_sym @cse_read
+ Scenario: Read an object with CSE symmetric with metadata specified on both ends
+ Given I ask for the object with key "foo"
+ And I have a symmetric "128" CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I read it with the CSE key
+ And I specify metadata CSE material location
+ Then the result should be "HELLO"
+
+ @cse_sym @cse_write
+ Scenario: Write an object with CSE symmetric with instruction file material location on both ends
+ Given I ask for the object with key "foo"
+ And I have a symmetric "256" CSE key
+ And I specify instruction file CSE material location
+ When I write the string "HELLO" to it using CSE
+ And I specify instruction file CSE material location
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with an instruction file
+ And the object should have unencrypted_content_length
+
+# @cse_sym @cse_read
+# Scenario: Read an object with incorrect material location at runtime (will not find encryption materials)
+# Given I ask for the object with key "foo"
+# And I have a symmetric "192" CSE key
+# And I specify metadata CSE material location
+# And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+# When I specify instruction file CSE material location
+# And I read it with the CSE key
+# Then the result should not be "HELLO"
+
+
+ @cse_asym @cse_read @cse_multipart
+ Scenario: Perform a multipart upload with CSE
+ Given the multipart upload threshold is 5mb
+ And I have a 7mb file
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ When I write the file to the object "foo" with CSE and the following metadata:
+ | color | red |
+ | shape | circle |
+ Then the metadata of object "foo" should eventually include:
+ | color | red |
+ | shape | circle |
+ Then the file at key "foo" should eventually be encrypted
+ And the contents of CSE object "foo" should eventually match the file
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_asym @cse_read @cse_block
+ Scenario: Performing a streaming read with a block and CSE
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I read it with the CSE key and a block into a variable
+ Then the result should be "HELLO"
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_asym @cse_read @cse_multipart @cse_block
+ Scenario: Performing a streaming read with a block and CSE and a large file uploaded with multipart
+ Given the multipart upload threshold is 5mb
+ And I have a 7mb file
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the CSE encrypted file
+ When I read it with the CSE key
+ Then the contents of CSE object "foo" should eventually match the file
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_sym @cse_read @cse_multipart @cse_block
+ Scenario: Performing a streaming read with a block and CSE and a large file uploaded with multipart and instruction file material location using a symmetric key
+ Given the multipart upload threshold is 5mb
+ And I have a 7mb file
+ And I have a symmetric "256" CSE key
+ And I specify instruction file CSE material location
+ And in the bucket the object with key "foo" has the CSE encrypted file
+ When I specify instruction file CSE material location
+ And I read it with the CSE key
+ Then the contents of CSE object "foo" should eventually match the file
+ And the object CSE data should be stored with an instruction file
+ And the object should have unencrypted_content_length
+
+ @cse_asym @cse_copy
+ Scenario: Copy an object with CSE
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I copy "foo" to "foo2" with CSE enabled
+ And I ask for the object with key "foo2"
+ Then the object should eventually have encrypted "HELLO" as its body
+ And the object CSE data should be stored with metadata
+ And the object should have unencrypted_content_length
+
+ @cse_asym @cse_copy
+ Scenario: Copy an object with CSE and decrypt
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify metadata CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I copy "foo" to "foo2" with CSE enabled
+ And I specify metadata CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+
+ @cse_asym @cse_copy
+ Scenario: Copy an object with CSE and decrypt with instruction filematerial location
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify instruction file CSE material location
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I copy "foo" to "foo2" with CSE enabled
+ And I specify instruction file CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+ And the object CSE data should be stored with an instruction file
+
+
+ @cse_asym @cse_copy
+ Scenario: Move an object with CSE and decrypt with instruction filematerial location and md5
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify instruction file CSE material location
+ And I specify "blah blah" as the md5
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I move "foo" to "foo2" with CSE enabled
+ And I specify instruction file CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+ And the object CSE data should be stored with an instruction file
+
+ @cse_asym @cse_copy
+ Scenario: Copy an object with CSE and decrypt with instruction filematerial location and no md5
+ Given I ask for the object with key "foo"
+ And I have an asymmetric CSE key
+ And I specify instruction file CSE material location
+ And I specify no md5
+ And in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
+ When I copy "foo" to "foo2" with CSE enabled
+ And I specify instruction file CSE material location
+ And I read it with the CSE key
+ Then the result should be "HELLO"
+ And the object CSE data should be stored with an instruction file
+
+
+# @cse_read
+# Scenario: Read a file with CSE that wasn't encrypted (will produce error)
+# Given I ask for the object with key "foo"
+# And I have an asymmetric CSE key
+# And I write the string "HELLO" to it
+# Then I specify metadata CSE material location
+# And I read it with the CSE key
+# And the result should be "HELLO"
+
+ @cse_write
+ Scenario: Write a file using an IO-like object
+ Given the multipart upload threshold is 5mb
+ And I have a 2mb file
+ And I put the file in an IO object
+ And I have an asymmetric CSE key
+ And I ask for the object with key "foo"
+ When I specify metadata CSE material location
+ And I write the IO object
+ Then the contents of CSE object "foo" should eventually match the file
+
+ @cse_write
+ Scenario: Write a file using an IO-like object multipart
+ Given the multipart upload threshold is 5mb
+ And I have a 7mb file
+ And I put the file in an IO object
+ And I have an asymmetric CSE key
+ And I ask for the object with key "foo"
+ When I specify metadata CSE material location
+ And I write the IO object
+ Then the contents of CSE object "foo" should eventually match the file
+
+
+ @cse_write @test
+ Scenario: Write an IO like object with an estimated length of 5mb
+ Given the multipart upload threshold is 10mb
+ And I have a 7mb file
+ And I have an asymmetric CSE key
+ And I ask for the object with key "foo"
+ And I specify metadata CSE material location
+ When I write the file using an IO-like object and an estimated length of 5mb
+ Then the contents of CSE object "foo" should eventually match the file
+
+ @cse_delete
+ Scenario: Delete an object uploaded with CSE and an instruction file
+ Given the multipart upload threshold is 5mb
+ And I have a 7mb file
+ And I put the file in an IO object
+ And I have an asymmetric CSE key
+ And I ask for the object with key "foo"
+ And I specify metadata CSE material location
+ And I write the IO object
+ When I delete the CSE object with :delete_instruction_file
+ Then No instruction file remains
+
+
View
5 features/s3/high_level/objects.feature
@@ -87,11 +87,6 @@ Feature: CRUD Objects (High Level)
When I write the string "HELLO" to it
Then the result should be the object with key "foo"
And the object should eventually have "HELLO" as its body
- And a request should have been made like:
- | TYPE | NAME | VALUE |
- | http | verb | PUT |
- | http | uri | /foo |
- | http | body | HELLO |
@put_object @multibyte
Scenario: Write an object with a multibyte string
View
235 features/s3/high_level/step_definitions/client_side_encryption.rb
@@ -0,0 +1,235 @@
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+# language: en
+
+When /^I write the string "([^"]*)" to it using CSE$/ do |contents|
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:content_md5] = @md5 unless @md5.nil?
+
+ @object.write(contents, options)
+end
+
+Then /^the object should eventually have encrypted "([^"]*)" as its body$/ do |data|
+ @result = @object.read()
+ @result.should_not eq(data)
+end
+
+Then /^the object CSE data should be stored with metadata$/ do
+ @result = @object.metadata.to_h
+ @result['x-amz-key'].should_not eq(nil)
+ @result['x-amz-iv'].should_not eq(nil)
+ @result['x-amz-matdesc'].should_not eq(nil)
+end
+
+Then /^the object should have unencrypted_content_length$/ do
+ @object.metadata['x-amz-unencrypted-content-length'].should_not eq(nil)
+end
+
+Given /^in the bucket the object with key "([^"]*)" has the contents CSE encrypted "([^"]*)"$/ do |key, data|
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:content_md5] = @md5 unless @md5.nil?
+ @bucket.objects[key].write(data, options)
+end
+
+Given /^I have an asymmetric CSE key$/ do
+ @cse_key = OpenSSL::PKey::RSA.new(2048)
+end
+
+
+Given /^I have a symmetric "([^"]*)" CSE key$/ do |num|
+ a_cipher = OpenSSL::Cipher.new("AES-#{num}-CBC")
+ a_cipher.encrypt
+ @cse_key = a_cipher.random_key
+end
+
+When /^I read it with the CSE key$/ do
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ @result = @object.read(options)
+end
+
+Given /^I specify no CSE material description$/ do
+ @cse_matdesc = nil
+end
+
+Given /^I specify the CSE material description as:$/ do |table|
+ @cse_matdesc = table.rows_hash
+end
+
+Given /^I specify no CSE material location$/ do
+ @cse_material_location = nil
+end
+
+Given /^I specify metadata CSE material location$/ do
+ @cse_material_location = :metadata
+end
+
+Given /^I specify instruction file CSE material location$/ do
+ @cse_material_location = :instruction_file
+end
+
+Then /^the object CSE data should be stored with an instruction file$/ do
+ @object.metadata['x-amz-key'].should eq(nil)
+ @object.metadata['x-amz-iv'].should eq(nil)
+ @object.metadata['x-amz-matdesc'].should eq(nil)
+ cse_inst_file = @bucket.objects["#{@object.key}.instruction"]
+ cse_inst_file.exists?.should eq(true)
+ cse_inst_file.metadata['x-amz-crypto-instr-file'].should eq("")
+ json_file = cse_inst_file.read()
+ begin
+ data = JSON.parse(json_file)
+ data['x-amz-key'].should_not eq(nil)
+ data['x-amz-iv'].should_not eq(nil)
+ #Rescue?
+ end
+end
+
+Then /^the result should not be "([^"]*)"$/ do |data|
+ @result.should_not eq(data)
+end
+
+When /^I specify "([^"]*)" as the md5$/ do |md5|
+ @md5 = md5
+end
+
+When /^I specify no md5$/ do
+ @md5 = nil
+end
+
+When /^I write the file to the object "([^\"]*)" with CSE and the following metadata:$/ do |key, table|
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:content_md5] = @md5 unless @md5.nil?
+ options[:data] = @file.open
+ options[:metadata] = table.rows_hash
+ @object = @bucket.objects[key]
+ @object.write(options)
+end
+
+Then /^the metadata of object "([^\"]*)" should eventually include:$/ do |key, table|
+ object = @bucket.objects[key]
+ sleep 1 until object.head.content_length > 0
+ meta = object.head.meta.to_h
+ array = meta.keys
+ table.rows_hash { |key, value| meta[key].should eq(value) }
+end
+
+
+Then /^the file at key "([^\"]*)" should eventually be encrypted$/ do |key|
+ @result = @object.read()
+ @result.should_not eq(@file.open)
+end
+
+
+Given /^in the bucket the object with key "([^"]*)" has the CSE encrypted file$/ do |key|
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:content_md5] = @md5 unless @md5.nil?
+ options[:data] = @file.open
+ @object = @bucket.objects[key]
+ @object.write(options)
+end
+
+Then /^the contents of CSE object "([^\"]*)" should eventually match the file$/ do |key|
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ sleep 1
+ object = @bucket.objects[key]
+ Digest::MD5.hexdigest(object.read(options)).should ==
+ Digest::MD5.file(@file.path).hexdigest
+end
+
+When /^I read it with the CSE key and a block into a variable$/ do
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ @result = ""
+ @object.read(options) { |output| @result += output }
+end
+
+When /^I copy "([^"]*)" to "([^"]*)" with CSE enabled$/ do |key1, key2|
+ options = {}
+ options[:client_side_encrypted] = true
+ options[:metadata] = {:garblegarblegarble => "replace metadata"}
+ @bucket.objects[key1].copy_to(key2, options)
+ @object = @bucket.objects[key2]
+end
+
+When /^I move "([^"]*)" to "([^"]*)" with CSE enabled$/ do |key1, key2|
+ options = {}
+ options[:client_side_encrypted] = true
+ options[:metadata] = {:garblegarblegarble => "replace metadata"}
+ @bucket.objects[key1].move_to(key2, options)
+ @object = @bucket.objects[key2]
+end
+
+Given /^I put the file in an IO object$/ do
+ @IO_object = @file.open
+end
+
+When /^I write the IO object$/ do
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:data] = @IO_object
+ @object.write(options)
+end
+
+When /^I write the file using an IO-like object and an estimated length of (\d+)mb$/ do |length|
+ @fd = @file.open
+ class IOwithoutsize
+ def initialize io
+ @io = io
+ end
+
+ def read bytes
+ @io.read(bytes)
+ end
+
+ def eof?
+ @io.eof?
+ end
+ end
+
+ options = {}
+ options[:encryption_key] = @cse_key unless @cse_key.nil?
+ options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
+ options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
+ options[:estimated_content_length] = length.to_i * 1024 * 1024
+ options[:data] = IOwithoutsize.new(@file.open)
+ @object.write(options)
+end
+
+When /^I delete the CSE object with :delete_instruction_file$/ do
+ @object.delete(:delete_instruction_file => true)
+end
+
+Then /^No instruction file remains$/ do
+ @object.bucket.objects[@object.key + '.instruction'].exists?.should == false
+end
View
2  features/s3/high_level/step_definitions/objects.rb
@@ -290,7 +290,7 @@ def meta_hash table
Given /^I have the following objects:$/ do |table|
table.hashes.each do |hash|
- @bucket.objects[hash['KEY']].write(hash['DATA'])
+ @bucket.objects[hash['KEY']].write(hash['DATA'] || '')
end
end
View
8 features/s3/high_level/step_definitions/versions.rb
@@ -36,11 +36,15 @@
end
When /^the bucket should not be versioned$/ do
- @bucket.versioned?.should == false
+ eventually do
+ @bucket.versioned?.should == false
+ end
end
When /^the bucket versioning state should be "([^"]*)"$/ do |state|
- @bucket.versioning_state.should == state.to_sym
+ eventually do
+ @bucket.versioning_state.should == state.to_sym
+ end
end
Then /^there should be (\d+) versions for the object$/ do |count|
View
4 features/s3/high_level/versions.feature
@@ -16,7 +16,7 @@
Feature: High-Level Bucket and Object Versioning
As a user of the high-level S3 interface
- I want to work with versioned buckets and objects
+ I want to work with versioned buckets and objects
So that I can store multiple versions of objects
@bucket
@@ -24,7 +24,7 @@ Feature: High-Level Bucket and Object Versioning
When I create a new bucket
Then the bucket should not be versioned
And the bucket versioning state should be "unversioned"
-
+
@bucket
Scenario: Enable bucket versioning
Given I create a new bucket
View
8 features/support/common.rb
@@ -11,6 +11,9 @@
# ANY KIND, either express or implied. See the License for the specific
# language governing permissions and limitations under the License.
+require 'simplecov'
+SimpleCov.start
+
$: << File.join(File.dirname(File.dirname(File.dirname(__FILE__))), "lib")
require 'aws'
@@ -21,6 +24,7 @@
require 'uri'
require 'yaml'
+
# find a config file
dir = Dir.getwd
while dir != "/" and
@@ -60,9 +64,9 @@
AfterConfiguration do
AWS.config(test_config)
- handler = AWS::Core::Http::Handler.new(AWS.config.http_handler) do |req, resp|
+ handler = AWS::Core::Http::Handler.new(AWS.config.http_handler) do |req, resp, read_block|
(@requests_made ||= []) << req
- super(req, resp)
+ super(req, resp, &read_block)
@last_response = resp
end
class << handler
View
15 lib/aws/core.rb
@@ -69,7 +69,7 @@
module AWS
# Current version of the AWS SDK for Ruby
- VERSION = "1.6.2"
+ VERSION = "1.6.3"
register_autoloads(self) do
autoload :Errors, 'errors'
@@ -299,7 +299,7 @@ class << self
# +true+, requests will always use path style. This can be useful
# for testing environments.
#
- # @option options [Integer] :s3_multipart_max_parts (1000) The maximum
+ # @option options [Integer] :s3_multipart_max_parts (10000) The maximum
# number of parts to split a file into when uploading in parts to S3.
#
# @option options [Integer] :s3_multipart_threshold (16777216) When
@@ -328,6 +328,17 @@ class << self
# * {S3::S3Object#presigned_post}
# * {S3::Bucket#presigned_post}
#
+ # @option options [OpenSSL::PKey::RSA, String] :s3_encryption_key (nil)
+ # If this is set, AWS::S3::S3Object #read and #write methods will always
+ # perform client-side encryption with this key. The key can be overridden
+ # at runtime by using the :encryption_key option. A value of nil
+ # means that client-side encryption will not be used.
+ #
+ # @option options [Symbol] :s3_encryption_materials_location (:metadata)
+ # When set to +:instruction_file+, AWS::S3::S3Object will store
+ # encryption materials in a seperate object, instead of the object
+ # metadata.
+ #
# @option options [String] :simple_db_endpoint ('sdb.amazonaws.com')
# The service endpoint for Amazon SimpleDB.
#
View
2  lib/aws/core/autoloader.rb
@@ -42,7 +42,7 @@ module Core
# @private
class Autoloader
-
+
def initialize klass, prefix = nil
@klass = klass
@prefix = prefix || klass.name.gsub(/::/, '/').downcase
View
99 lib/aws/core/client.rb
@@ -21,6 +21,11 @@ module Core
# Base client class for all of the Amazon AWS service clients.
class Client
+ # Raised when a request failed due to a networking issue (e.g.
+ # EOFError, IOError, Errno::ECONNRESET, Errno::EPIPE,
+ # Timeout::Error, etc)
+ class NetworkError < StandardError; end
+
extend Naming
# @private
@@ -155,6 +160,19 @@ def new_stub_for method_name
response
end
+ # Logs the warning to the configured logger, otherwise to stderr.
+ # @param [String] warning
+ # @return [nil]
+ def log_warning warning
+ message = '[aws-sdk-gem-warning] ' + warning
+ if config.logger
+ config.logger.warn(message)
+ else
+ $stderr.puts(message)
+ end
+ nil
+ end
+
protected
def new_request
@@ -204,13 +222,30 @@ def async_request_with_retries response, http_request, retry_delays = nil
end
- def make_sync_request response
+ def make_sync_request response, &read_block
retry_server_errors do
- response.http_response = http_response =
- Http::Response.new
+ response.http_response = Http::Response.new
+
+ @http_handler.handle(
+ response.http_request,
+ response.http_response,
+ &read_block)
+
+ if
+ block_given? and
+ response.http_response.status < 300 and
+ response.http_response.body
+ then
+
+ msg = ":http_handler read the entire http response body into "
+ msg << "memory, it should have instead yielded chunks"
+ log_warning(msg)
+
+ # go ahead and yield the body on behalf of the handler
+ yield(response.http_response.body)
- @http_handler.handle(response.http_request, http_response)
+ end
populate_error(response)
response.signal_success unless response.error
@@ -227,7 +262,6 @@ def retry_server_errors &block
while should_retry?(response)
break if sleeps.empty?
Kernel.sleep(sleeps.shift)
- # rebuild the request to get a fresh signature
rebuild_http_request(response)
response = yield
end
@@ -256,8 +290,16 @@ def scaling_factor response
end
def should_retry? response
+ if retryable_error?(response)
+ response.safe_to_retry?
+ else
+ false
+ end
+ end
+
+ def retryable_error? response
expired_credentials?(response) or
- response.timeout? or
+ response.network_error? or
response.throttled? or
response.error.kind_of?(Errors::ServerError)
end
@@ -303,7 +345,7 @@ def log_client_request options, &block
end
# Logs the response to the configured logger.
- # @param [Resposne] response
+ # @param [Response] response
# @return [nil]
def log_response response
if config.logger
@@ -336,10 +378,10 @@ def extract_error response
]
case
- when response.timeout? then TimeoutError.new
- when error_code then error_class(error_code).new(*error_args)
- when status >= 500 then Errors::ServerError.new(*error_args)
- when status >= 300 then Errors::ClientError.new(*error_args)
+ when response.network_error? then NetworkError.new
+ when error_code then error_class(error_code).new(*error_args)
+ when status >= 500 then Errors::ServerError.new(*error_args)
+ when status >= 300 then Errors::ClientError.new(*error_args)
else nil # no error
end
@@ -371,21 +413,25 @@ def errors_module
AWS.const_get(self.class.to_s[/(\w+)::Client/, 1])::Errors
end
- def client_request name, options, &block
+ def client_request name, options, &read_block
return_or_raise(options) do
log_client_request(options) do
if config.stub_requests?
response = stub_for(name)
- response.http_request = build_request(name, options, &block)
+ response.http_request = build_request(name, options)
response.request_options = options
response
else
client = self
- response = new_response { client.send(:build_request, name, options, &block) }
+
+ response = new_response do
+ client.send(:build_request, name, options)
+ end
+
response.request_type = name
response.request_options = options
@@ -399,8 +445,8 @@ def client_request name, options, &block
else
# process the http request
options[:async] ?
- make_async_request(response) :
- make_sync_request(response)
+ make_async_request(response, &read_block) :
+ make_sync_request(response, &read_block)
# process the http response
response.on_success do
@@ -424,7 +470,7 @@ def cacheable_request? name, options
self.class::CACHEABLE_REQUESTS.include?(name)
end
- def build_request(name, options, &block)
+ def build_request name, options
# we dont want to pass the async option to the configure block
opts = options.dup
@@ -445,7 +491,7 @@ def build_request(name, options, &block)
http_request.ssl_ca_file = config.ssl_ca_file if config.ssl_ca_file
http_request.ssl_ca_path = config.ssl_ca_path if config.ssl_ca_path
- send("configure_#{name}_request", http_request, opts, &block)
+ send("configure_#{name}_request", http_request, opts)
http_request.headers["user-agent"] = user_agent_string
http_request.add_authorization!(credential_provider)
@@ -474,16 +520,18 @@ def user_agent_string
#
def self.add_client_request_method method_name, options = {}, &block
- self.operations << method_name
+ operations << method_name
ClientRequestMethodBuilder.new(self, method_name, &block)
- module_eval <<-END
+ method_def = <<-METHOD
def #{method_name}(*args, &block)
options = args.first ? args.first : {}
client_request(#{method_name.inspect}, options, &block)
end
- END
+ METHOD
+
+ module_eval(method_def)
end
@@ -518,15 +566,6 @@ def initialize client_class, method_name, &block
def configure_request options = {}, &block
name = "configure_#{@method_name}_request"
MetaUtils.class_extend_method(@client_class, name, &block)
- if block.arity == 3
- m = Module.new
- m.module_eval(<<-END)
- def #{name}(req, options, &block)
- super(req, options, block)
- end
- END
- @client_class.send(:include, m)
- end
end
def process_response &block
View
13 lib/aws/core/configuration.rb
@@ -128,7 +128,7 @@ module Core
# +true+, requests will always use path style. This can be useful
# for testing environments.
#
- # @attr_reader [Integer] s3_multipart_max_parts (1000)
+ # @attr_reader [Integer] s3_multipart_max_parts (10000)
# The maximum number of parts to split a file into when uploading
# in parts to S3.
#
@@ -163,6 +163,17 @@ module Core
#
# s3 = AWS::S3.new(:s3_server_side_encryption => :aes256)
#
+ # @attr_reader [OpenSSL::PKey::RSA, String] s3_encryption_key
+ # If this is set, AWS::S3::S3Object #read and #write methods will always
+ # perform client-side encryption with this key. The key can be overridden
+ # at runtime by using the :encryption_key option. A value of nil
+ # means that client-side encryption will not be used.
+ #
+ # @attr_reader [Symbol] s3_encryption_materials_location
+ # When set to +:instruction_file+, AWS::S3::S3Object will store
+ # encryption materials in a seperate object, instead of the object
+ # metadata.
+ #
# @attr_reader [String] simple_db_endpoint ('sdb.amazonaws.com')
# The service endpoint for Amazon SimpleDB.
#
View
44 lib/aws/core/http/handler.rb
@@ -14,27 +14,39 @@
module AWS
module Core
module Http
-
+
# @private
class Handler
-
+
attr_reader :base
-
+
def initialize(base, &block)
@base = base
if base.respond_to?(:handle)
-
- unless block.arity == 2
- raise ArgumentError, 'passed block must accept 2 arguments'
+
+ unless [2,3].include?(block.arity)
+ raise ArgumentError, 'passed block must accept 2 or 3 arguments'
end
+
MetaUtils.extend_method(self, :handle, &block)
-
+
+ if block.arity == 3
+ m = Module.new do
+ eval(<<-DEF)
+ def handle req, resp, &read_block
+ super(req, resp, read_block)
+ end
+ DEF
+ end
+ self.extend(m)
+ end
+
elsif base.respond_to?(:handle_async)
-
+
unless block.arity == 3
raise ArgumentError, 'passed block must accept 3 arguments'
end
-
+
MetaUtils.extend_method(self, :handle_async) do |req, resp, handle|
@base.handle_async(req, resp, handle)
end
@@ -44,16 +56,16 @@ def initialize(base, &block)
end
define_method(:handle_async, &block)
end
-
+
else
raise ArgumentError, 'base must respond to #handle or #handle_async'
end
end
-
- def handle(request, http_response)
- @base.handle(request, http_response)
+
+ def handle(request, http_response, &read_block)
+ @base.handle(request, http_response, &read_block)
end
-
+
def handle_async(request, http_response, handle)
Thread.new do
begin
@@ -65,12 +77,12 @@ def handle_async(request, http_response, handle)
end
end
end
-
+
def sleep_with_callback seconds, &block
Kernel.sleep(seconds)
yield
end
-
+
end
end
end
View
42 lib/aws/core/http/net_http_handler.rb
@@ -25,6 +25,18 @@ module Http
#
class NetHttpHandler
+ # @private
+ NETWORK_ERRORS = [
+ EOFError,
+ IOError,
+ Errno::ECONNABORTED,
+ Errno::ECONNRESET,
+ Errno::EPIPE,
+ Errno::EINVAL,
+ Timeout::Error,
+ Errno::ETIMEDOUT,
+ ]
+
# (see Net::HTTP::ConnectionPool.new)
def initialize options = {}
@pool = Net::HTTP::ConnectionPool.new(options)
@@ -39,7 +51,7 @@ def initialize options = {}
# @param [Request] request
# @param [Response] response
# @return [nil]
- def handle request, response
+ def handle request, response, &read_block
options = {}
options[:port] = request.port
@@ -49,17 +61,25 @@ def handle request, response
options[:ssl_ca_file] = request.ssl_ca_file if request.ssl_ca_file
options[:ssl_ca_path] = request.ssl_ca_path if request.ssl_ca_path
- connection = pool.connection_for(request.host, options)
- connection.read_timeout = request.read_timeout
-
begin
- http_response = connection.request(build_net_http_request(request))
- response.body = http_response.body
- response.status = http_response.code.to_i
- response.headers = http_response.to_hash
- rescue Timeout::Error, Errno::ETIMEDOUT => e
- response.timeout = true
+
+ connection = pool.connection_for(request.host, options)
+ connection.read_timeout = request.read_timeout
+
+ connection.request(build_net_http_request(request)) do |http_resp|
+ response.status = http_resp.code.to_i
+ response.headers = http_resp.to_hash
+ if block_given? and response.status < 300
+ http_resp.read_body(&read_block)
+ else
+ response.body = http_resp.read_body
+ end
+ end
+
+ rescue *NETWORK_ERRORS
+ response.network_error = true
end
+
nil
end
@@ -90,7 +110,7 @@ def build_net_http_request request
end
net_http_req = request_class.new(request.uri, headers)
- net_http_req.body = request.body
+ net_http_req.body_stream = request.body_stream
net_http_req
end
View
68 lib/aws/core/http/request.rb
@@ -36,7 +36,7 @@ def initialize
# 60 seconds.
attr_accessor :default_read_timeout
- # @return [String] Returns hostname of the request.
+ # @return [String] hostname of the request
attr_accessor :host
# @return [Integer] Returns the port number this request will be
@@ -47,7 +47,7 @@ def initialize
# 'POST', 'HEAD' or 'DELETE'). Defaults to 'POST'.
attr_accessor :http_method
- # @return [Hash] Returns a hash of header values.
+ # @return [CaseInsensitiveHash] request headers
attr_accessor :headers
# @return [String] Returns the request URI (path + querystring).
@@ -57,24 +57,18 @@ def initialize
# to be populated for requests against signature v4 endpoints.
attr_accessor :region
- # @return [String]
+ # @return [String] Returns the AWS access key ID used to authorize the
+ # request.
# @private
attr_accessor :access_key_id
- # @private
# @return [Array<Param>] Returns an array of request params. Requests
# that use signature version 2 add params to the request and then
# sign those before building the {#body}. Normally the {#body}
# should be set directly with the HTTP payload.
+ # @private
attr_accessor :params
- # @return [String] Returns the HTTP request payload (body).
- attr_accessor :body
-
- # @return [String] Returns the AWS access key ID used to authorize the
- # request.
- attr_accessor :access_key_id
-
# @return [String] The name of the service for Signature v4 signing.
# This does not always match the ruby name (e.g.
# simple_email_service and ses do not match).
@@ -125,11 +119,6 @@ def read_timeout
default_read_timeout
end
- # @return [String,nil] Returns the request body (payload).
- def body
- @body || url_encoded_params
- end
-
# @return [String] Returns the HTTP request path.
def path
uri.split(/\?/)[0]
@@ -167,6 +156,53 @@ def url_encoded_params
params.empty? ? nil : params.sort.collect(&:encoded).join('&')
end
+ # @param [String] body
+ def body= body
+ @body = body
+ if body
+ headers['content-length'] = body.size if body
+ else
+ headers.delete('content-length')
+ end
+ end
+
+ # @note Calling #body on a request with a #body_stream
+ # will cause the entire stream to be read into memory.
+ # @return [String,nil] Returns the request body.
+ def body
+ if @body
+ @body
+ elsif @body_stream
+ @body = @body_stream.read
+ if @body_stream.respond_to?(:rewind)
+ @body_stream.rewind
+ else
+ @body_stream = StringIO.new(@body)
+ end
+ @body
+ else
+ nil
+ end
+ end
+
+ # Sets the request body as an IO object that will be streamed.
+ # @note You must also set the #headers['content-length']
+ # @param [IO] stream An object that responds to #read and #eof.
+ def body_stream= stream
+ @body_stream = stream
+ end
+
+ # @return [IO,nil]
+ def body_stream
+ if @body_stream
+ @body_stream
+ elsif @body
+ StringIO.new(@body)
+ else
+ nil
+ end
+ end
+
# @private
class CaseInsensitiveHash < Hash
View
36 lib/aws/core/http/response.rb
@@ -14,7 +14,7 @@
module AWS
module Core
module Http
-
+
# Represents the http response from a service request.
#
# Responses have:
@@ -23,22 +23,26 @@ module Http
# * headers (hash of response headers)
# * body (the response body)
class Response
-
- # @return [Integer] (200) response http status code
+
+ # @return [Integer] Returns the http response status code.
attr_accessor :status
-
- # @return [Hash] ({}) response http headers
+
+ # @return [Hash] ({}) Returns the HTTP response headers.
attr_accessor :headers
-
- # @return [String] ('') response http body
+
+ # @return [String,nil] Returns the HTTP response body.
attr_accessor :body
-
- # @return [Boolean] (false) set to true if the client gives up
- # before getting a response from the service.
- attr_accessor :timeout
- alias_method :timeout?, :timeout
-
+ # @return [Boolean] Returns +true+ if the request could not be made
+ # because of a networking issue (including timeouts).
+ attr_accessor :network_error
+
+ alias_method :network_error?, :network_error
+
+ # The #network_error attribute was previously #timeout, aliasing
+ # for backwards compatability
+ alias_method :timeout=, :network_error=
+
# @param [Hash] options
# @option options [Integer] :status (200) HTTP status code
# @option options [Hash] :headers ({}) HTTP response headers
@@ -46,11 +50,11 @@ class Response
def initialize options = {}, &block
@status = options[:status] || 200
@headers = options[:headers] || {}
- @body = options[:body] || ''
+ @body = options[:body]
yield(self) if block_given?
self
end
-
+
# Returns the header value with the given name.
#
# The value is matched case-insensitively so if the headers hash
@@ -67,7 +71,7 @@ def header name
end
nil
end
-
+
end
end
end
View
28 lib/aws/core/indifferent_hash.rb
@@ -14,7 +14,7 @@
module AWS
module Core
- # A utility class to provide indifferent access to hash data.
+ # A utility class to provide indifferent access to hash data.
#
# Inspired by ActiveSupport's HashWithIndifferentAccess, this class
# has a few notable differences:
@@ -26,10 +26,10 @@ module Core
# These features were omitted because our primary use for this class is to
# wrap a 1-level hash as a return value, but we want the user to access
# the values with string or symbol keys.
- #
+ #
# @private
class IndifferentHash < Hash
-
+
def initialize *args
if args.first.is_a?(Hash)
super()
@@ -38,51 +38,51 @@ def initialize *args
super(*args)
end
end
-
+
alias_method :_getter, :[]
alias_method :_setter, :[]=
-
+
def []=(key, value)
_setter(_convert_key(key), value)
end
alias_method :store, :[]=
-
+
def [] key
_getter(_convert_key(key))
end
-
+
def merge! hash
hash.each_pair do |key,value|
- self[key] = value
+ self[key] = value
end
self
end
alias_method :update, :merge!
-
+
def merge hash
self.dup.merge!(hash)
end
-
+
def has_key? key
super(_convert_key(key))
end
alias_method :key?, :has_key?
alias_method :member?, :has_key?
alias_method :include?, :has_key?
-
+
def fetch key, *extras, &block
super(_convert_key(key), *extras, &block)
end
-
+
def delete key
super(_convert_key(key))
end
-
+
private
def _convert_key key
key.is_a?(String) ? key : key.to_s
end
-
+
end
end
end
View
1  lib/aws/core/query_client.rb
@@ -93,6 +93,7 @@ def define_client_method method_name, operation_name
parser.request_params(options).each do |param|
request.add_param(param)
end
+ request.body = request.url_encoded_params
end
View
46 lib/aws/core/response.rb
@@ -16,14 +16,18 @@ module Core
# = Response
#
- # Each service request returns a response object. Responses provide
- # access to response data and request/response metadata.
+ # Each Service has a Client class. There is one method per service
+ # operation defined on the client. These methods all return a {Response}
+ # object.
+ #
+ # In addition to the response data, these responses provide metadata
+ # about the HTTP request made and the HTTP response received.
#
# == Response Data
#
- # Each response has a hash of data that represents the data
- # returned by the service. You can get at this data by
- # calling {#data} (you can also use the {#[]} method as a shortcut)
+ # You can access the response data for a client request using the {#data}
+ # method or the {#[]} method. Response data is a hash and {#[]} is
+ # a shortcut for accessing this hash.
#
# # make a request to describe one instance
# ec2 = AWS::EC2.new
@@ -40,10 +44,10 @@ module Core
# In addition to the response data, there is additional information
# available with the response, including:
#
- # * the name of the client request method called
- # * the hash of options passed to the client request
- # * the HTTP request object (useful for debugging)
- # * the HTTP response object (useful for debugging)
+ # * {#request_type} - the name of the client request method
+ # * {#request_options} - the hash of options passed to the client method
+ # * {#http_request} - The HTTP request made
+ # * {#http_response} - the HTTP response received
#
# Given the example and response object from above:
#
@@ -101,7 +105,7 @@ def initialize http_request = nil, http_response = nil, &block
@data = {}
@retry_count = 0
@duration = 0
- rebuild_request if @request_builder && !http_request
+ build_request if @request_builder && !http_request
end
# Provides access to the response data. This is a short-cut
@@ -130,9 +134,10 @@ def throttled?
end
end
- # @return [Boolean] Returns true if the http request timed out.
- def timeout?
- http_response.timeout?
+ # @return [Boolean] Returns +true+ if the http request failed due to
+ # a networking issue.
+ def network_error?
+ http_response.network_error?
end
# @return [String]
@@ -157,11 +162,24 @@ def cache_key
# (throttling, server errors, socket errors, etc).
# @private
def rebuild_request
- @http_request = @request_builder.call
+ build_request
+ @http_request.body_stream.rewind if @http_request.body_stream
+ end
+
+ # @return [Boolean] Returns +false+ if it is not safe to retry a
+ # request. This happens when the http request body is an IO
+ # object that can not be rewound and re-streamed.
+ def safe_to_retry?
+ @http_request.body_stream.nil? or
+ @http_request.body_stream.respond_to?(:rewind)
end
protected
+ def build_request
+ @http_request = @request_builder.call
+ end
+
# @note The prefered method to get as response data is to use {#[]}.
#
# This provides a backwards-compat layer to the old response objects
View
1  lib/aws/core/signature/version_2.rb
@@ -24,6 +24,7 @@ def add_authorization! credentials
add_param('SignatureVersion', '2')
add_param('SignatureMethod', 'HmacSHA256')
add_param('Signature', signature(credentials))
+ self.body = url_encoded_params
end
protected
View
32 lib/aws/core/signature/version_4.rb
@@ -19,7 +19,7 @@ module AWS
module Core
module Signature
module Version4
-
+
def self.included base
base.send(:include, Signer)
end
@@ -29,13 +29,13 @@ def add_authorization! credentials
headers['content-type'] ||= 'application/x-www-form-urlencoded'
headers['host'] = host
headers['x-amz-date'] = datetime
- headers['x-amz-security-token'] = credentials.session_token if
+ headers['x-amz-security-token'] = credentials.session_token if
credentials.session_token
headers['authorization'] = authorization(credentials, datetime)
end
-
+
protected
-
+
def authorization credentials, datetime
parts = []
parts << "AWS4-HMAC-SHA256 Credential=#{credentials.access_key_id}/#{credential_string(datetime)}"
@@ -43,7 +43,7 @@ def authorization credentials, datetime
parts << "Signature=#{hex16(signature(credentials, datetime))}"
parts.join(', ')
end
-
+
def signature credentials, datetime
k_secret = credentials.secret_access_key
k_date = hmac("AWS4" + k_secret, datetime[0,8])
@@ -52,7 +52,7 @@ def signature credentials, datetime
k_credentials = hmac(k_service, 'aws4_request')
hmac(k_credentials, string_to_sign(datetime))
end
-
+
def string_to_sign datetime
parts = []
parts << 'AWS4-HMAC-SHA256'
@@ -61,8 +61,8 @@ def string_to_sign datetime
parts << hex16(hash(canonical_request))
parts.join("\n")
end
-
- def credential_string datetime
+
+ def credential_string datetime
parts = []
parts << datetime[0,8]
parts << region
@@ -70,7 +70,7 @@ def credential_string datetime
parts << 'aws4_request'
parts.join("/")
end
-
+
def canonical_request
parts = []
parts << http_method
@@ -81,18 +81,18 @@ def canonical_request
parts << hex16(hash(body || ''))
parts.join("\n")
end
-
+
def service
# this method is implemented in the request class for each service
raise NotImplementedError
end
-
+
def signed_headers
to_sign = headers.keys.map{|k| k.to_s.downcase }
to_sign.delete('authorization')
to_sign.sort.join(";")
end
-
+
def canonical_headers
headers = []
self.headers.each_pair do |k,v|
@@ -101,20 +101,20 @@ def canonical_headers
headers = headers.sort_by(&:first)
headers.map{|k,v| "#{k}:#{canonical_header_values(v)}" }.join("\n")
end
-
+
def canonical_header_values values
values = [values] unless values.is_a?(Array)
values.map(&:to_s).map(&:strip).join(',')
end
-
+
def hex16 string
string.unpack('H*').first
end
-
+
def hash string
Digest::SHA256.digest(string)
end
-
+
end
end
end
View
4 lib/aws/dynamo_db/client.rb
@@ -761,7 +761,7 @@ def extract_error_details response
end
end
- def should_retry? response
+ def retryable_error? response
if response.error.is_a?(Errors::ProvisionedThroughputExceededException)
config.dynamo_db_retry_throughput_errors?
else
@@ -771,7 +771,7 @@ def should_retry? response
def sleep_durations response
- retry_count =
+ retry_count =
if expired_credentials?(response)
config.max_retries == 0 ? 0 : 1
else
View
6 lib/aws/dynamo_db/request.rb
@@ -13,20 +13,14 @@
module AWS
class DynamoDB
-
# @private
class Request < Core::Http::Request
-
include Core::Signature::Version4
def service
'dynamodb'
end
- # @return [String,nil]
- attr_accessor :body
-
end
-
end
end
View
20 lib/aws/rails.rb
@@ -35,7 +35,7 @@ class Railtie < Rails::Railtie
# for rails 2 and bundler for rails 3) then {setup} is called
# automatically.
module Rails
-
+
# Adds extra functionality to Rails.
#
# Normailly this method is invoked automatically when you require this
@@ -56,13 +56,13 @@ def self.setup
log_to_rails_logger
nil
end
-
+
# Loads AWS configuration options from +RAILS_ROOT/config/aws.yml+.
#
# This configuration file is optional. You can omit this file and instead
# use ruby to configure AWS inside a configuration initialization script
# (e.g. RAILS_ROOT/config/intializers/aws.rb).
- #
+ #
# If you have a yaml configuration file it should be formatted like the
# standard +database.yml+ file in a Rails application. This means there
# should be one section for Rails environment:
@@ -76,8 +76,8 @@ def self.setup
# access_key_id: YOUR_ACCESS_KEY_ID
# secret_access_key: YOUR_SECRET_ACCESS_KEY
# simple_db_consistent_reads: true
- #
- # You should also consider DRYing up your configuration file using
+ #
+ # You should also consider DRYing up your configuration file using
# YAML references:
#
# development:
@@ -90,7 +90,7 @@ def self.setup
# simple_db_consistent_reads: true
#
# The yaml file will also be ERB parsed so you can use ruby inside of it:
- #
+ #
# development:
# access_key_id: YOUR_ACCESS_KEY_ID
# secret_access_key: <%= read_secret_from_a_secure_location %>
@@ -101,9 +101,9 @@ def self.setup
# simple_db_consistent_reads: true
#
def self.load_yaml_config
-
+
path = Pathname.new("#{rails_root}/config/aws.yml")
-
+
if File.exists?(path)
cfg = YAML::load(ERB.new(File.read(path)).result)
unless cfg[rails_env]
@@ -111,9 +111,9 @@ def self.load_yaml_config
end
AWS.config(cfg[rails_env])
end
-
+
end
-
+
# Adds a delivery method to ActionMailer that uses
# {AWS::SimpleEmailService}.
#
View
73 lib/aws/s3.rb
@@ -26,7 +26,7 @@ module AWS
# * {Amazon S3}[http://aws.amazon.com/s3/]
# * {Amazon S3 Documentation}[http://aws.amazon.com/documentation/s3/]
#
- # == Credentials
+ # = Credentials
#
# You can setup default credentials for all AWS services via
# AWS.config:
@@ -34,63 +34,76 @@ module AWS
# AWS.config(
# :access_key_id => 'YOUR_ACCESS_KEY_ID',
# :secret_access_key => 'YOUR_SECRET_ACCESS_KEY')
- #
+ #
# Or you can set them directly on the S3 interface:
#
# s3 = AWS::S3.new(
# :access_key_id => 'YOUR_ACCESS_KEY_ID',
# :secret_access_key => 'YOUR_SECRET_ACCESS_KEY')
#
- # == Buckets Keys and Objects
+ # = Buckets
#
- # S3 stores objects in buckets.
+ # Before you can upload files to S3, you need to create a bucket.
#
- # To create a bucket:
+ # s3 = AWS::S3.new
+ # bucket = s3.buckets.create('my-bucket')
#
- # bucket = s3.buckets.create('mybucket')
+ # If a bucket already exists, you can get a reference to the bucket.
#
- # To get a bucket:
+ # bucket = s3.buckets['my-bucket'] # no request made
#
- # bucket = s3.buckets[:mybucket]
- # bucket = s3.buckets['mybucket']
+ # You can also enumerate all buckets in your account.
#
- # Listing buckets:
- #
# s3.buckets.each do |bucket|
# puts bucket.name
# end
#
- # See {Bucket} and {BucketCollection} for more information on working
- # with S3 buckets.
+ # See {BucketCollection} and {Bucket} for more information on working
+ # with buckets.
#
- # == Listing Objects
+ # = Objects
#
- # Enumerating objects in a bucket:
+ # Buckets contain objects. Each object in a bucket has a unique key.
#
- # bucket.objects.each do |object|
- # puts object.key #=> no data is fetched from s3, just a list of keys
- # end
+ # == Getting an Object
+ #
+ # If the object already exists, you can get a reference to the object.
#
- # You can limit the list of objects with a prefix, or explore the objects
- # in a bucket as a tree. See {ObjectCollection} for more information.
+ # # makes no request, returns an AWS::S3::S3Object
+ # obj = bucket.objects['key']
#
- # == Reading and Writing to S3
+ # == Reading and Writing an Object
#
- # Each object in a bucket has a unique key.
+ # The example above returns an {S3Object}. You call {S3Object#write} and
+ # {S3Object#read} to upload to and download from S3 respectively.
#
- # photo = s3.buckets['mybucket'].objects['photo.jpg']
+ # # streaming upload a file to S3
+ # obj.write(Pathname.new('/path/to/file.txt'))
#
- # Writing to an S3Object:
+ # # streaming download from S3 to a file on disk
+ # File.open('file.txt', 'w') do |file|
+ # obj.read do |chunk|
+ # file.write(chunk)
+ # end
+ # end
+ #
+ # == Enumerating Objects
#
- # photo.write(File.read('/some/photo.jpg'))
+ # You can enumerate objects in your buckets.
#
- # Reading from an S3Object:
+ # # enumerate ALL objects in the bucket (even if the bucket contains
+ # # more than 1k objects)
+ # bucket.objects.each do |obj|
+ # puts obj.key
+ # end
#
- # File.open("/some/path/on/disk.jpg", "w") do |f|
- # f.write(photo.read)
+ # # enumerate at most 20 objects with the given prefix
+ # bucket.objects.with_prefix('photos/').each(:limit => 20).each do |photo|
+ # puts photo.key
# end
#
- # See {S3Object} for more information on reading and writing to S3.
+ # See {ObjectCollection} and {S3Object} for more information on working
+ # with objects.
#
class S3
@@ -104,6 +117,8 @@ class S3
autoload :BucketVersionCollection, 'bucket_version_collection'
autoload :Client, 'client'
autoload :DataOptions, 'data_options'
+ autoload :EncryptionUtils, 'encryption_utils'
+ autoload :CipherIO, 'cipher_io'
autoload :Errors, 'errors'
autoload :MultipartUpload, 'multipart_upload'
autoload :MultipartUploadCollection, 'multipart_upload_collection'
View
177 lib/aws/s3/bucket.rb
@@ -14,15 +14,180 @@
module AWS
class S3
- # Represents a single S3 bucket.
+ # Represents a bucket in S3.
#
- # @example Creating a Bucket
- #
- # bucket = s3.buckets.create('mybucket')
+ # = Creating Buckets
#
- # @example Getting an Existing Bucket
+ # You create a bucket by name. Bucket names must be globally unique
+ # and must be DNS compatible.
#
- # bucket = s3.buckets['mybucket']
+ # s3 = AWS::S3.new
+ # bucket = s3.buckets.create('dns-compat-bucket-name')
+ #
+ # = Getting a Bucket
+ #
+ # You can create a reference to a bucket, given its name.
+ #
+ # bucket = s3.buckets['bucket-name'] # makes no request
+ # bucket.exists? #=> returns true/false
+ #
+ # = Enumerating Buckets
+ #
+ # The {BucketCollection} class is enumerable.
+ #
+ # s3.buckets.each do |bucket|
+ # puts bucket.name
+ # end
+ #
+ # = Deleting a Bucket
+ #
+ # You can delete an empty bucket you own.
+ #
+ # bucket = s3.buckets.create('my-temp-bucket')
+ # bucket.objects['abc'].write('xyz')
+ #
+ # bucket.clear! # deletes all object versions in batches
+ # bucket.delete
+ #
+ # You can alternatively call {#delete!} which will clear
+ # the bucket for your first.
+ #
+ # bucket.delete!
+ #
+ # = Objects
+ #
+ # Given a bucket you can access its objects, either by key or by
+ # enumeration.
+ #
+ # bucket.objects['key'] #=> makes no request, returns an S3Object
+ #
+ # bucket.objects.each do |obj|
+ # puts obj.key
+ # end
+ #
+ # See {ObjectCollection} and {S3Object} for more information on working
+ # with objects.
+ #
+ # = Bucket Policies and ACLs
+ #
+ # You can control access to your bucket and its contents a number
+ # of ways. You can specify a bucket ACL (access control list)
+ # or a bucket policy.
+ #
+ # == ACLs
+ #
+ # ACLs control access to your bucket and its contents via a list of
+ # grants and grantees.
+ #
+ # === Canned ACLs
+ #
+ # The simplest way to specify an ACL is to use one of Amazon's "canned"
+ # ACLs. Amazon accepts the following canned ACLs:
+ #
+ # * +:private+
+ # * +:public_read+
+ # * +:public_read_write+
+ # * +:authenticated_read+
+ # * +:bucket_owner_read+
+ # * +:bucket_owner_full_control+
+ #
+ # You can specify a the ACL at bucket creation or later update a bucket.
+ #
+ # # at create time, defaults to :private when not specified
+ # bucket = s3.buckets.create('name', :acl => :public_read)
+ #
+ # # replacing an existing bucket ACL
+ # bucket.acl = :private
+ #
+ # === Grants
+ #
+ # Alternatively you can specify a hash of grants. Each entry in the
+ # +:grant+ hash has a grant (key) and a list of grantees (values).
+ # Valid grant keys are:
+ #
+ # * +:grant_read+
+ # * +:grant_write+
+ # * +:grant_read_acp+
+ # * +:grant_write_acp+
+ # * +:grant_full_control+
+ #
+ # Each grantee can be a String, Hash or array of strings or hashes.
+ # The following example uses grants to provide public read
+ # to everyone while providing full control to a user by email address
+ # and to another by their account id (cannonical user id).
+ #
+ # bucket = s3.buckets.create('name', :grants => {
+ # :grant_read => [
+ # { :uri => "http://acs.amazonaws.com/groups/global/AllUsers" },
+ # ],
+ # :grant_full_control => [
+ # { :id => 'abc...mno' } # cannonical user id
+ # { :email_address => 'foo@bar.com' }, # email address
+ # ]
+ # })
+ #
+ # === ACL Object
+ #
+ # Lastly, you can build an ACL object and use a Ruby DSL to specify grants
+ # and grantees. See {ACLObject} for more information.
+ #
+ # # updating an existing bucket acl using ACLObject
+ # bucket.acl.change do |acl|
+ # acl.grants.reject! do |g|
+ # g.grantee.canonical_user_id != bucket.owner.id
+ # end
+ # end
+ #
+ # == Policies
+ #
+ # You can also work with bucket policies.
+ #
+ # policy = AWS::S3::Policy.new
+ # policy.allow(
+ # :actions => [:put_object, :get_object]
+ # :resources => [bucket]
+ # :principals => :any)
+ #
+ # bucket.policy = policy
+ #
+ # See {Core::Policy} and {S3::Policy} for more information on build
+ # policy objects.
+ #
+ # = Versioned Buckets
+ #
+ # You can enable versioning on a bucket you control. When versioning
+ # is enabled, S3 will keep track of each version of each object you
+ # write to the bucket (even deletions).
+ #
+ # bucket.versioning_enabled? #=> false
+ # bucket.enable_versioning
+ # # there is also a #disable_versioning method
+ #
+ # obj = bucket.objects['my-obj']
+ # obj.write('a')
+ # obj.write('b')
+ # obj.delete
+ # obj.write('c')
+ #
+ # obj.versions.each do |obj_version|
+ # if obj_version.delete_marker?
+ # puts obj_version.read
+ # else
+ # puts "- DELETE MARKER"
+ # end
+ # end
+ #
+ # Alternatively you can enumerate all versions of all objects in your
+ # bucket.
+ #
+ # bucket.versions.each do |obj_version|
+ # puts obj_version.key + " : " + obj_version.version_id
+ # end
+ #
+ # See {BucketVersionCollection}, {ObjectVersionCollection} and
+ # {ObjectVersion} for more information on working with objects in
+ # a versioned bucket. Also see the S3 documentation for information
+ # on object versioning.
#
class Bucket
View
119 lib/aws/s3/cipher_io.rb
@@ -0,0 +1,119 @@
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#