Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Version 1.6.3 of the AWS SDK for Ruby

  • Loading branch information...
commit 9877f13179fdffaa31484416ecf690d52597a767 1 parent b91a8ba
Trevor Rowe authored

Showing 58 changed files with 3,661 additions and 883 deletions. Show diff stats Hide diff stats

  1. 1  Gemfile
  2. 3  features/ec2/step_definitions/ec2.rb
  3. 10  features/ec2/step_definitions/snapshot_attributes.rb
  4. 326  features/s3/high_level/client_side_encryption.feature
  5. 5  features/s3/high_level/objects.feature
  6. 235  features/s3/high_level/step_definitions/client_side_encryption.rb
  7. 2  features/s3/high_level/step_definitions/objects.rb
  8. 8  features/s3/high_level/step_definitions/versions.rb
  9. 4  features/s3/high_level/versions.feature
  10. 8  features/support/common.rb
  11. 15  lib/aws/core.rb
  12. 2  lib/aws/core/autoloader.rb
  13. 99  lib/aws/core/client.rb
  14. 13  lib/aws/core/configuration.rb
  15. 44  lib/aws/core/http/handler.rb
  16. 42  lib/aws/core/http/net_http_handler.rb
  17. 68  lib/aws/core/http/request.rb
  18. 36  lib/aws/core/http/response.rb
  19. 28  lib/aws/core/indifferent_hash.rb
  20. 1  lib/aws/core/query_client.rb
  21. 46  lib/aws/core/response.rb
  22. 1  lib/aws/core/signature/version_2.rb
  23. 32  lib/aws/core/signature/version_4.rb
  24. 4  lib/aws/dynamo_db/client.rb
  25. 6  lib/aws/dynamo_db/request.rb
  26. 20  lib/aws/rails.rb
  27. 73  lib/aws/s3.rb
  28. 177  lib/aws/s3/bucket.rb
  29. 119  lib/aws/s3/cipher_io.rb
  30. 120  lib/aws/s3/client.rb
  31. 6  lib/aws/s3/config.rb
  32. 185  lib/aws/s3/data_options.rb
  33. 144  lib/aws/s3/encryption_utils.rb
  34. 14  lib/aws/s3/errors.rb
  35. 11  lib/aws/s3/multipart_upload.rb
  36. 4  lib/aws/s3/object_collection.rb
  37. 2  lib/aws/s3/policy.rb
  38. 54  lib/aws/s3/request.rb
  39. 1,034  lib/aws/s3/s3_object.rb
  40. 2  lib/aws/simple_email_service/request.rb
  41. 3  lib/aws/simple_workflow/request.rb
  42. 138  lib/net/http/connection_pool.rb
  43. 84  lib/net/http/connection_pool/connection.rb
  44. 45  lib/net/http/connection_pool/session.rb
  45. 2  spec/aws/config_spec.rb
  46. 33  spec/aws/core/client_spec.rb
  47. 4  spec/aws/core/http/httparty_handler_spec.rb
  48. 44  spec/aws/core/http/net_http_handler_spec.rb
  49. 60  spec/aws/core/http/request_spec.rb
  50. 4  spec/aws/core/http/response_spec.rb
  51. 31  spec/aws/core/response_spec.rb
  52. 115  spec/aws/s3/cipher_io_spec.rb
  53. 100  spec/aws/s3/client_spec.rb
  54. 674  spec/aws/s3/s3_object_encrypt_spec.rb
  55. 121  spec/aws/s3/s3_object_spec.rb
  56. 27  spec/net/http/connection_pool/session_spec.rb
  57. 52  spec/shared/aws_client_examples.rb
  58. 3  spec/spec_helper.rb
1  Gemfile
@@ -39,6 +39,7 @@ group :build do
39 39
   gem 'rspec', '2.5'
40 40
   gem 'rspec', '2.5', :require => 'rspec/core/rake_task'
41 41
   gem 'rcov', '0.9.9'
  42
+  gem 'simplecov', :require => false
42 43
   gem 'ci_reporter', '~> 1.6', :require => 'ci/reporter/rake/rspec'
43 44
 end
44 45
 
3  features/ec2/step_definitions/ec2.rb
@@ -131,7 +131,8 @@
131 131
   @created_volumes.each do |volume|
132 132
     begin
133 133
       volume.delete
134  
-    rescue AWS::EC2::Errors::InvalidVolumeID::NotFound
  134
+    rescue AWS::EC2::Errors::InvalidVolumeID::NotFound,
  135
+      AWS::EC2::Errors::InvalidVolume::NotFound
135 136
       # already deleted
136 137
     end
137 138
   end
10  features/ec2/step_definitions/snapshot_attributes.rb
@@ -41,9 +41,17 @@
41 41
 end
42 42
 
43 43
 Given /^I create a snapshot with description "([^\"]*)"$/ do |description|
  44
+
44 45
   Given %(I create a volume)
45  
-  @snapshot = @volume.create_snapshot(description)
  46
+
  47
+  # it can take a while before the volume is in a state where snapshots 
  48
+  # are allowed
  49
+  eventually do
  50
+    @snapshot = @volume.create_snapshot(description)
  51
+  end
  52
+
46 53
   @created_snapshots << @snapshot
  54
+
47 55
 end
48 56
 
49 57
 Then /^the snapshot description should be "([^\"]*)"$/ do |description|
326  features/s3/high_level/client_side_encryption.feature
... ...
@@ -0,0 +1,326 @@
  1
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2
+#
  3
+# Licensed under the Apache License, Version 2.0 (the "License"). You
  4
+# may not use this file except in compliance with the License. A copy of
  5
+# the License is located at
  6
+#
  7
+#     http://aws.amazon.com/apache2.0/
  8
+#
  9
+# or in the "license" file accompanying this file. This file is
  10
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11
+# ANY KIND, either express or implied. See the License for the specific
  12
+# language governing permissions and limitations under the License.
  13
+
  14
+# language: en
  15
+@s3 @high_level @objects @cse
  16
+Feature: Amazon S3 Client Side Encryption
  17
+
  18
+  As a customer of Amazon S3
  19
+  I want my data to be envelope encrypted within Amazon S3
  20
+  So that I can store my private data securely while managing my own key
  21
+
  22
+  @cse_asym @cse_write
  23
+  Scenario: Write an object with CSE
  24
+    Given I ask for the object with key "foo"
  25
+    And   I have an asymmetric CSE key
  26
+    And   I specify no CSE material location
  27
+    When  I write the string "HELLO" to it using CSE
  28
+    And   I specify metadata CSE material location
  29
+    Then  the object should eventually have encrypted "HELLO" as its body
  30
+    And   the object CSE data should be stored with metadata
  31
+    And   the object should have unencrypted_content_length
  32
+
  33
+  @cse_asym @cse_read
  34
+  Scenario: Read an object with CSE
  35
+    Given I ask for the object with key "foo"
  36
+    And   I have an asymmetric CSE key
  37
+    And   I specify no CSE material location
  38
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  39
+    When  I specify metadata CSE material location
  40
+    And   I read it with the CSE key
  41
+    Then  the result should be "HELLO"
  42
+
  43
+  @cse_asym @cse_read
  44
+  Scenario: Read an object with CSE with metadata specified
  45
+    Given I ask for the object with key "foo"
  46
+    And   I have an asymmetric CSE key
  47
+    And   I specify metadata CSE material location
  48
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  49
+    When  I read it with the CSE key
  50
+    And   I specify metadata CSE material location
  51
+    Then  the result should be "HELLO"
  52
+
  53
+  @cse_asym @cse_write
  54
+  Scenario: Write an object with CSE with instruction file material location
  55
+    Given I ask for the object with key "foo"
  56
+    And   I have an asymmetric CSE key
  57
+    And   I specify instruction file CSE material location
  58
+    When  I write the string "HELLO" to it using CSE
  59
+    And   I specify instruction file CSE material location
  60
+    Then  the object should eventually have encrypted "HELLO" as its body
  61
+    And   the object CSE data should be stored with an instruction file
  62
+    And   the object should have unencrypted_content_length
  63
+
  64
+#  @cse_asym @cse_read
  65
+#  Scenario: Read an object with CSE with instruction file material location when not specified
  66
+#    Given I ask for the object with key "foo"
  67
+#    And   I have an asymmetric CSE key
  68
+#    And   I specify instruction file CSE material location
  69
+#    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  70
+#    When  I read it with the CSE key
  71
+#    And   I specify no CSE material location
  72
+#    Then  the result should be "HELLO"
  73
+
  74
+  @cse_asym @cse_read
  75
+  Scenario: Read an object with CSE with instruction file material location when specified
  76
+    Given I ask for the object with key "foo"
  77
+    And   I have an asymmetric CSE key
  78
+    And   I specify instruction file CSE material location
  79
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  80
+    When  I read it with the CSE key
  81
+    Then  the result should be "HELLO"
  82
+
  83
+  @cse_sym @cse_write
  84
+  Scenario: Write an object with CSE symmetric
  85
+    Given I ask for the object with key "foo"
  86
+    And   I have a symmetric "128" CSE key
  87
+    And   I specify no CSE material location
  88
+    When  I write the string "HELLO" to it using CSE
  89
+    Then  the object should eventually have encrypted "HELLO" as its body
  90
+    And   the object CSE data should be stored with metadata
  91
+    And   the object should have unencrypted_content_length
  92
+
  93
+  @cse_sym @cse_read
  94
+  Scenario: Read an object with CSE symmetric
  95
+    Given I ask for the object with key "foo"
  96
+    And   I have a symmetric "192" CSE key
  97
+    And   I specify no CSE material location
  98
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  99
+    When  I specify metadata CSE material location
  100
+    And   I read it with the CSE key
  101
+    Then  the result should be "HELLO"
  102
+
  103
+  @cse_sym @cse_read
  104
+  Scenario: Read an object with CSE symmetric with metadata specified
  105
+    Given I ask for the object with key "foo"
  106
+    And   I have a symmetric "256" CSE key
  107
+    And   I specify metadata CSE material location
  108
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  109
+    When  I read it with the CSE key
  110
+    And   I specify metadata CSE material location
  111
+    Then  the result should be "HELLO"
  112
+
  113
+  @cse_sym @cse_write
  114
+  Scenario: Write an object with CSE symmetric with instruction file material location
  115
+    Given I ask for the object with key "foo"
  116
+    And   I have a symmetric "192" CSE key
  117
+    And   I specify instruction file CSE material location
  118
+    When  I write the string "HELLO" to it using CSE
  119
+    And   I specify instruction file CSE material location
  120
+    Then  the object should eventually have encrypted "HELLO" as its body
  121
+    And   the object CSE data should be stored with an instruction file
  122
+    And   the object should have unencrypted_content_length
  123
+
  124
+  @cse_sym @cse_read
  125
+  Scenario: Read an object with CSE symmetric with metadata specified on both ends
  126
+    Given I ask for the object with key "foo"
  127
+    And   I have a symmetric "128" CSE key
  128
+    And   I specify metadata CSE material location
  129
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  130
+    When  I read it with the CSE key
  131
+    And   I specify metadata CSE material location
  132
+    Then  the result should be "HELLO"
  133
+
  134
+  @cse_sym @cse_write
  135
+  Scenario: Write an object with CSE symmetric with instruction file material location on both ends
  136
+    Given I ask for the object with key "foo"
  137
+    And   I have a symmetric "256" CSE key
  138
+    And   I specify instruction file CSE material location
  139
+    When  I write the string "HELLO" to it using CSE
  140
+    And   I specify instruction file CSE material location
  141
+    Then  the object should eventually have encrypted "HELLO" as its body
  142
+    And   the object CSE data should be stored with an instruction file
  143
+    And   the object should have unencrypted_content_length
  144
+
  145
+#  @cse_sym @cse_read
  146
+#  Scenario: Read an object with incorrect material location at runtime (will not find encryption materials)
  147
+#    Given I ask for the object with key "foo"
  148
+#    And   I have a symmetric "192" CSE key
  149
+#    And   I specify metadata CSE material location
  150
+#    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  151
+#    When  I specify instruction file CSE material location
  152
+#    And   I read it with the CSE key
  153
+#    Then  the result should not be "HELLO"
  154
+
  155
+
  156
+  @cse_asym @cse_read @cse_multipart
  157
+  Scenario: Perform a multipart upload with CSE
  158
+    Given the multipart upload threshold is 5mb
  159
+    And   I have a 7mb file
  160
+    And   I have an asymmetric CSE key
  161
+    And   I specify metadata CSE material location
  162
+    When  I write the file to the object "foo" with CSE and the following metadata:
  163
+    | color | red    |
  164
+    | shape | circle |
  165
+    Then the metadata of object "foo" should eventually include:
  166
+    | color | red    |
  167
+    | shape | circle |
  168
+    Then the file at key "foo" should eventually be encrypted
  169
+    And  the contents of CSE object "foo" should eventually match the file
  170
+    And  the object CSE data should be stored with metadata
  171
+    And  the object should have unencrypted_content_length
  172
+
  173
+  @cse_asym @cse_read @cse_block
  174
+  Scenario: Performing a streaming read with a block and CSE
  175
+    Given I ask for the object with key "foo"
  176
+    And   I have an asymmetric CSE key
  177
+    And   I specify metadata CSE material location
  178
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  179
+    When  I read it with the CSE key and a block into a variable
  180
+    Then  the result should be "HELLO"
  181
+    And   the object CSE data should be stored with metadata
  182
+    And   the object should have unencrypted_content_length
  183
+
  184
+  @cse_asym @cse_read @cse_multipart @cse_block
  185
+  Scenario: Performing a streaming read with a block and CSE and a large file uploaded with multipart
  186
+    Given the multipart upload threshold is 5mb
  187
+    And   I have a 7mb file
  188
+    And   I have an asymmetric CSE key
  189
+    And   I specify metadata CSE material location
  190
+    And   in the bucket the object with key "foo" has the CSE encrypted file
  191
+    When  I read it with the CSE key
  192
+    Then  the contents of CSE object "foo" should eventually match the file
  193
+    And   the object CSE data should be stored with metadata
  194
+    And   the object should have unencrypted_content_length
  195
+
  196
+  @cse_sym @cse_read @cse_multipart @cse_block
  197
+  Scenario: Performing a streaming read with a block and CSE and a large file uploaded with multipart and instruction file material location using a symmetric key
  198
+    Given the multipart upload threshold is 5mb
  199
+    And   I have a 7mb file
  200
+    And   I have a symmetric "256" CSE key
  201
+    And   I specify instruction file CSE material location
  202
+    And   in the bucket the object with key "foo" has the CSE encrypted file
  203
+    When  I specify instruction file CSE material location
  204
+    And   I read it with the CSE key
  205
+    Then  the contents of CSE object "foo" should eventually match the file
  206
+    And   the object CSE data should be stored with an instruction file
  207
+    And   the object should have unencrypted_content_length
  208
+
  209
+  @cse_asym @cse_copy
  210
+  Scenario: Copy an object with CSE
  211
+    Given I ask for the object with key "foo"
  212
+    And   I have an asymmetric CSE key
  213
+    And   I specify metadata CSE material location
  214
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  215
+    When  I copy "foo" to "foo2" with CSE enabled
  216
+    And   I ask for the object with key "foo2"
  217
+    Then  the object should eventually have encrypted "HELLO" as its body
  218
+    And   the object CSE data should be stored with metadata
  219
+    And   the object should have unencrypted_content_length
  220
+
  221
+  @cse_asym @cse_copy
  222
+  Scenario: Copy an object with CSE and decrypt
  223
+    Given I ask for the object with key "foo"
  224
+    And   I have an asymmetric CSE key
  225
+    And   I specify metadata CSE material location
  226
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  227
+    When  I copy "foo" to "foo2" with CSE enabled
  228
+    And   I specify metadata CSE material location
  229
+    And   I read it with the CSE key
  230
+    Then  the result should be "HELLO"
  231
+
  232
+  @cse_asym @cse_copy
  233
+  Scenario: Copy an object with CSE and decrypt with instruction filematerial location
  234
+    Given I ask for the object with key "foo"
  235
+    And   I have an asymmetric CSE key
  236
+    And   I specify instruction file CSE material location
  237
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  238
+    When  I copy "foo" to "foo2" with CSE enabled
  239
+    And   I specify instruction file CSE material location
  240
+    And   I read it with the CSE key
  241
+    Then  the result should be "HELLO"
  242
+    And   the object CSE data should be stored with an instruction file
  243
+
  244
+
  245
+  @cse_asym @cse_copy
  246
+  Scenario: Move an object with CSE and decrypt with instruction filematerial location and md5
  247
+    Given I ask for the object with key "foo"
  248
+    And   I have an asymmetric CSE key
  249
+    And   I specify instruction file CSE material location
  250
+    And   I specify "blah blah" as the md5
  251
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  252
+    When  I move "foo" to "foo2" with CSE enabled
  253
+    And   I specify instruction file CSE material location
  254
+    And   I read it with the CSE key
  255
+    Then  the result should be "HELLO"
  256
+    And   the object CSE data should be stored with an instruction file
  257
+
  258
+  @cse_asym @cse_copy
  259
+  Scenario: Copy an object with CSE and decrypt with instruction filematerial location and no md5
  260
+    Given I ask for the object with key "foo"
  261
+    And   I have an asymmetric CSE key
  262
+    And   I specify instruction file CSE material location
  263
+    And   I specify no md5
  264
+    And   in the bucket the object with key "foo" has the contents CSE encrypted "HELLO"
  265
+    When  I copy "foo" to "foo2" with CSE enabled
  266
+    And   I specify instruction file CSE material location
  267
+    And   I read it with the CSE key
  268
+    Then  the result should be "HELLO"
  269
+    And   the object CSE data should be stored with an instruction file
  270
+
  271
+
  272
+#  @cse_read
  273
+#  Scenario: Read a file with CSE that wasn't encrypted (will produce error)
  274
+#    Given I ask for the object with key "foo"
  275
+#    And   I have an asymmetric CSE key
  276
+#    And   I write the string "HELLO" to it
  277
+#    Then  I specify metadata CSE material location
  278
+#    And   I read it with the CSE key
  279
+#    And   the result should be "HELLO"
  280
+
  281
+  @cse_write
  282
+  Scenario: Write a file using an IO-like object
  283
+    Given the multipart upload threshold is 5mb
  284
+    And   I have a 2mb file
  285
+    And   I put the file in an IO object
  286
+    And   I have an asymmetric CSE key
  287
+    And   I ask for the object with key "foo"
  288
+    When  I specify metadata CSE material location
  289
+    And   I write the IO object
  290
+    Then  the contents of CSE object "foo" should eventually match the file
  291
+
  292
+  @cse_write
  293
+  Scenario: Write a file using an IO-like object multipart
  294
+    Given the multipart upload threshold is 5mb
  295
+    And   I have a 7mb file
  296
+    And   I put the file in an IO object
  297
+    And   I have an asymmetric CSE key
  298
+    And   I ask for the object with key "foo"
  299
+    When  I specify metadata CSE material location
  300
+    And   I write the IO object
  301
+    Then  the contents of CSE object "foo" should eventually match the file
  302
+
  303
+
  304
+  @cse_write @test
  305
+  Scenario: Write an IO like object with an estimated length of 5mb
  306
+    Given the multipart upload threshold is 10mb
  307
+    And   I have a 7mb file
  308
+    And   I have an asymmetric CSE key
  309
+    And   I ask for the object with key "foo"
  310
+    And   I specify metadata CSE material location
  311
+    When  I write the file using an IO-like object and an estimated length of 5mb
  312
+    Then  the contents of CSE object "foo" should eventually match the file
  313
+
  314
+  @cse_delete
  315
+  Scenario: Delete an object uploaded with CSE and an instruction file
  316
+    Given the multipart upload threshold is 5mb
  317
+    And   I have a 7mb file
  318
+    And   I put the file in an IO object
  319
+    And   I have an asymmetric CSE key
  320
+    And   I ask for the object with key "foo"
  321
+    And   I specify metadata CSE material location
  322
+    And   I write the IO object
  323
+    When  I delete the CSE object with :delete_instruction_file
  324
+    Then  No instruction file remains
  325
+
  326
+
5  features/s3/high_level/objects.feature
@@ -87,11 +87,6 @@ Feature: CRUD Objects (High Level)
87 87
     When I write the string "HELLO" to it
88 88
     Then the result should be the object with key "foo"
89 89
     And the object should eventually have "HELLO" as its body
90  
-    And a request should have been made like:
91  
-    | TYPE | NAME | VALUE |
92  
-    | http | verb | PUT   |
93  
-    | http | uri  | /foo  |
94  
-    | http | body | HELLO |
95 90
 
96 91
   @put_object @multibyte
97 92
   Scenario: Write an object with a multibyte string
235  features/s3/high_level/step_definitions/client_side_encryption.rb
... ...
@@ -0,0 +1,235 @@
  1
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  2
+#
  3
+# Licensed under the Apache License, Version 2.0 (the "License"). You
  4
+# may not use this file except in compliance with the License. A copy of
  5
+# the License is located at
  6
+#
  7
+#     http://aws.amazon.com/apache2.0/
  8
+#
  9
+# or in the "license" file accompanying this file. This file is
  10
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
  11
+# ANY KIND, either express or implied. See the License for the specific
  12
+# language governing permissions and limitations under the License.
  13
+# language: en
  14
+
  15
+When /^I write the string "([^"]*)" to it using CSE$/ do |contents|
  16
+  options = {}
  17
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  18
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  19
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  20
+  options[:content_md5] = @md5 unless @md5.nil?
  21
+
  22
+  @object.write(contents, options)
  23
+end
  24
+
  25
+Then /^the object should eventually have encrypted "([^"]*)" as its body$/ do |data|
  26
+  @result = @object.read()
  27
+  @result.should_not eq(data)
  28
+end
  29
+
  30
+Then /^the object CSE data should be stored with metadata$/ do
  31
+  @result = @object.metadata.to_h
  32
+  @result['x-amz-key'].should_not eq(nil)
  33
+  @result['x-amz-iv'].should_not eq(nil)
  34
+  @result['x-amz-matdesc'].should_not eq(nil)
  35
+end
  36
+
  37
+Then /^the object should have unencrypted_content_length$/ do
  38
+  @object.metadata['x-amz-unencrypted-content-length'].should_not eq(nil)
  39
+end
  40
+
  41
+Given /^in the bucket the object with key "([^"]*)" has the contents CSE encrypted "([^"]*)"$/ do |key, data|
  42
+  options = {}
  43
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  44
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  45
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  46
+  options[:content_md5] = @md5 unless @md5.nil?
  47
+  @bucket.objects[key].write(data, options)
  48
+end
  49
+
  50
+Given /^I have an asymmetric CSE key$/ do
  51
+  @cse_key = OpenSSL::PKey::RSA.new(2048)
  52
+end
  53
+
  54
+
  55
+Given /^I have a symmetric "([^"]*)" CSE key$/ do |num|
  56
+  a_cipher = OpenSSL::Cipher.new("AES-#{num}-CBC")
  57
+  a_cipher.encrypt
  58
+  @cse_key = a_cipher.random_key
  59
+end
  60
+
  61
+When /^I read it with the CSE key$/ do
  62
+  options = {}
  63
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  64
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  65
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  66
+  @result = @object.read(options)
  67
+end
  68
+
  69
+Given /^I specify no CSE material description$/ do
  70
+  @cse_matdesc = nil
  71
+end
  72
+
  73
+Given /^I specify the CSE material description as:$/ do |table|
  74
+  @cse_matdesc = table.rows_hash
  75
+end
  76
+
  77
+Given /^I specify no CSE material location$/ do
  78
+  @cse_material_location = nil
  79
+end
  80
+
  81
+Given /^I specify metadata CSE material location$/ do
  82
+  @cse_material_location = :metadata
  83
+end
  84
+
  85
+Given /^I specify instruction file CSE material location$/ do
  86
+  @cse_material_location = :instruction_file
  87
+end
  88
+
  89
+Then /^the object CSE data should be stored with an instruction file$/ do
  90
+  @object.metadata['x-amz-key'].should eq(nil)
  91
+  @object.metadata['x-amz-iv'].should eq(nil)
  92
+  @object.metadata['x-amz-matdesc'].should eq(nil)
  93
+  cse_inst_file = @bucket.objects["#{@object.key}.instruction"]
  94
+  cse_inst_file.exists?.should eq(true)
  95
+  cse_inst_file.metadata['x-amz-crypto-instr-file'].should eq("")
  96
+  json_file = cse_inst_file.read()
  97
+  begin
  98
+    data = JSON.parse(json_file)
  99
+    data['x-amz-key'].should_not eq(nil)
  100
+    data['x-amz-iv'].should_not eq(nil)
  101
+  #Rescue?
  102
+  end
  103
+end
  104
+
  105
+Then /^the result should not be "([^"]*)"$/ do |data|
  106
+  @result.should_not eq(data)
  107
+end
  108
+
  109
+When /^I specify "([^"]*)" as the md5$/ do |md5|
  110
+  @md5 = md5
  111
+end
  112
+
  113
+When /^I specify no md5$/ do
  114
+  @md5 = nil
  115
+end
  116
+
  117
+When /^I write the file to the object "([^\"]*)" with CSE and the following metadata:$/ do |key, table|
  118
+  options = {}
  119
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  120
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  121
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  122
+  options[:content_md5] = @md5 unless @md5.nil?
  123
+  options[:data]        = @file.open
  124
+  options[:metadata]    = table.rows_hash
  125
+  @object = @bucket.objects[key]
  126
+  @object.write(options)
  127
+end
  128
+
  129
+Then /^the metadata of object "([^\"]*)" should eventually include:$/ do |key, table|
  130
+  object = @bucket.objects[key]
  131
+  sleep 1 until object.head.content_length > 0
  132
+  meta = object.head.meta.to_h
  133
+  array = meta.keys
  134
+  table.rows_hash { |key, value| meta[key].should eq(value) }
  135
+end
  136
+
  137
+
  138
+Then /^the file at key "([^\"]*)" should eventually be encrypted$/ do |key|
  139
+  @result = @object.read()
  140
+  @result.should_not eq(@file.open)
  141
+end
  142
+
  143
+
  144
+Given /^in the bucket the object with key "([^"]*)" has the CSE encrypted file$/ do |key|
  145
+  options = {}
  146
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  147
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  148
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  149
+  options[:content_md5] = @md5 unless @md5.nil?
  150
+  options[:data]        = @file.open
  151
+  @object = @bucket.objects[key]
  152
+  @object.write(options)
  153
+end
  154
+
  155
+Then /^the contents of CSE object "([^\"]*)" should eventually match the file$/ do |key|
  156
+  options = {}
  157
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  158
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  159
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  160
+  sleep 1
  161
+  object = @bucket.objects[key]
  162
+  Digest::MD5.hexdigest(object.read(options)).should ==
  163
+    Digest::MD5.file(@file.path).hexdigest
  164
+end
  165
+
  166
+When /^I read it with the CSE key and a block into a variable$/ do
  167
+  options = {}
  168
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  169
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  170
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  171
+  @result = ""
  172
+  @object.read(options) { |output| @result += output }
  173
+end
  174
+
  175
+When /^I copy "([^"]*)" to "([^"]*)" with CSE enabled$/ do |key1, key2|
  176
+  options = {}
  177
+  options[:client_side_encrypted] = true
  178
+  options[:metadata] = {:garblegarblegarble => "replace metadata"}
  179
+  @bucket.objects[key1].copy_to(key2, options)
  180
+  @object = @bucket.objects[key2]
  181
+end
  182
+
  183
+When /^I move "([^"]*)" to "([^"]*)" with CSE enabled$/ do |key1, key2|
  184
+  options = {}
  185
+  options[:client_side_encrypted] = true
  186
+  options[:metadata] = {:garblegarblegarble => "replace metadata"}
  187
+  @bucket.objects[key1].move_to(key2, options)
  188
+  @object = @bucket.objects[key2]
  189
+end
  190
+
  191
+Given /^I put the file in an IO object$/ do
  192
+  @IO_object = @file.open
  193
+end
  194
+
  195
+When /^I write the IO object$/ do
  196
+  options = {}
  197
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  198
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  199
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  200
+  options[:data] = @IO_object
  201
+  @object.write(options)
  202
+end
  203
+
  204
+When /^I write the file using an IO-like object and an estimated length of (\d+)mb$/ do |length|
  205
+  @fd = @file.open
  206
+  class IOwithoutsize
  207
+    def initialize io
  208
+      @io = io
  209
+    end
  210
+
  211
+    def read bytes
  212
+      @io.read(bytes)
  213
+    end
  214
+
  215
+    def eof?
  216
+      @io.eof?
  217
+    end
  218
+  end
  219
+
  220
+  options = {}
  221
+  options[:encryption_key] = @cse_key unless @cse_key.nil?
  222
+  options[:encryption_materials_location] = @cse_material_location unless @cse_material_location.nil?
  223
+  options[:encryption_matdesc] = @cse_matdesc unless @cse_matdesc.nil?
  224
+  options[:estimated_content_length] = length.to_i * 1024 * 1024
  225
+  options[:data] = IOwithoutsize.new(@file.open)
  226
+  @object.write(options)
  227
+end
  228
+
  229
+When /^I delete the CSE object with :delete_instruction_file$/ do
  230
+  @object.delete(:delete_instruction_file => true)
  231
+end
  232
+
  233
+Then /^No instruction file remains$/ do
  234
+  @object.bucket.objects[@object.key + '.instruction'].exists?.should == false
  235
+end
2  features/s3/high_level/step_definitions/objects.rb
@@ -290,7 +290,7 @@ def meta_hash table
290 290
 
291 291
 Given /^I have the following objects:$/ do |table|
292 292
   table.hashes.each do |hash|
293  
-    @bucket.objects[hash['KEY']].write(hash['DATA'])
  293
+    @bucket.objects[hash['KEY']].write(hash['DATA'] || '')
294 294
   end
295 295
 end
296 296
 
8  features/s3/high_level/step_definitions/versions.rb
@@ -36,11 +36,15 @@
36 36
 end
37 37
 
38 38
 When /^the bucket should not be versioned$/ do
39  
-  @bucket.versioned?.should == false
  39
+  eventually do
  40
+    @bucket.versioned?.should == false
  41
+  end
40 42
 end
41 43
 
42 44
 When /^the bucket versioning state should be "([^"]*)"$/ do |state|
43  
-  @bucket.versioning_state.should == state.to_sym
  45
+  eventually do
  46
+    @bucket.versioning_state.should == state.to_sym
  47
+  end
44 48
 end
45 49
 
46 50
 Then /^there should be (\d+) versions for the object$/ do |count|
4  features/s3/high_level/versions.feature
@@ -16,7 +16,7 @@
16 16
 Feature: High-Level Bucket and Object Versioning
17 17
 
18 18
   As a user of the high-level S3 interface
19  
-  I want to work with versioned buckets and objects 
  19
+  I want to work with versioned buckets and objects
20 20
   So that I can store multiple versions of objects
21 21
 
22 22
   @bucket
@@ -24,7 +24,7 @@ Feature: High-Level Bucket and Object Versioning
24 24
     When I create a new bucket
25 25
     Then the bucket should not be versioned
26 26
     And the bucket versioning state should be "unversioned"
27  
-  
  27
+
28 28
   @bucket
29 29
   Scenario: Enable bucket versioning
30 30
     Given I create a new bucket
8  features/support/common.rb
@@ -11,6 +11,9 @@
11 11
 # ANY KIND, either express or implied. See the License for the specific
12 12
 # language governing permissions and limitations under the License.
13 13
 
  14
+require 'simplecov'
  15
+SimpleCov.start
  16
+
14 17
 $: << File.join(File.dirname(File.dirname(File.dirname(__FILE__))), "lib")
15 18
 
16 19
 require 'aws'
@@ -21,6 +24,7 @@
21 24
 require 'uri'
22 25
 require 'yaml'
23 26
 
  27
+
24 28
 # find a config file
25 29
 dir = Dir.getwd
26 30
 while dir != "/" and
@@ -60,9 +64,9 @@
60 64
 
61 65
 AfterConfiguration do
62 66
   AWS.config(test_config)
63  
-  handler = AWS::Core::Http::Handler.new(AWS.config.http_handler) do |req, resp|
  67
+  handler = AWS::Core::Http::Handler.new(AWS.config.http_handler) do |req, resp, read_block|
64 68
     (@requests_made ||= []) << req
65  
-    super(req, resp)
  69
+    super(req, resp, &read_block)
66 70
     @last_response = resp
67 71
   end
68 72
   class << handler
15  lib/aws/core.rb
@@ -69,7 +69,7 @@
69 69
 module AWS
70 70
 
71 71
   # Current version of the AWS SDK for Ruby
72  
-  VERSION = "1.6.2"
  72
+  VERSION = "1.6.3"
73 73
 
74 74
   register_autoloads(self) do
75 75
     autoload :Errors, 'errors'
@@ -299,7 +299,7 @@ class << self
299 299
     #   +true+, requests will always use path style.  This can be useful
300 300
     #   for testing environments.
301 301
     #
302  
-    # @option options [Integer] :s3_multipart_max_parts (1000) The maximum
  302
+    # @option options [Integer] :s3_multipart_max_parts (10000) The maximum
303 303
     #   number of parts to split a file into when uploading in parts to S3.
304 304
     #
305 305
     # @option options [Integer] :s3_multipart_threshold (16777216) When
@@ -328,6 +328,17 @@ class << self
328 328
     #   * {S3::S3Object#presigned_post}
329 329
     #   * {S3::Bucket#presigned_post}
330 330
     #
  331
+    # @option options [OpenSSL::PKey::RSA, String] :s3_encryption_key (nil)
  332
+    #   If this is set, AWS::S3::S3Object #read and #write methods will always
  333
+    #   perform client-side encryption with this key. The key can be overridden
  334
+    #   at runtime by using the :encryption_key option.  A value of nil
  335
+    #   means that client-side encryption will not be used.
  336
+    #
  337
+    # @option options [Symbol] :s3_encryption_materials_location (:metadata)
  338
+    #   When set to +:instruction_file+, AWS::S3::S3Object will store
  339
+    #   encryption materials in a seperate object, instead of the object
  340
+    #   metadata.
  341
+    #
331 342
     # @option options [String] :simple_db_endpoint ('sdb.amazonaws.com')
332 343
     #   The service endpoint for Amazon SimpleDB.
333 344
     #
2  lib/aws/core/autoloader.rb
@@ -42,7 +42,7 @@ module Core
42 42
 
43 43
     # @private
44 44
     class Autoloader
45  
-      
  45
+
46 46
       def initialize klass, prefix = nil
47 47
         @klass = klass
48 48
         @prefix = prefix || klass.name.gsub(/::/, '/').downcase
99  lib/aws/core/client.rb
@@ -21,6 +21,11 @@ module Core
21 21
     # Base client class for all of the Amazon AWS service clients.
22 22
     class Client
23 23
 
  24
+      # Raised when a request failed due to a networking issue (e.g.
  25
+      # EOFError, IOError, Errno::ECONNRESET, Errno::EPIPE,
  26
+      # Timeout::Error, etc)
  27
+      class NetworkError < StandardError; end
  28
+
24 29
       extend Naming
25 30
 
26 31
       # @private
@@ -155,6 +160,19 @@ def new_stub_for method_name
155 160
         response
156 161
       end
157 162
 
  163
+      # Logs the warning to the configured logger, otherwise to stderr.
  164
+      # @param [String] warning
  165
+      # @return [nil]
  166
+      def log_warning warning
  167
+        message = '[aws-sdk-gem-warning] ' + warning
  168
+        if config.logger
  169
+          config.logger.warn(message)
  170
+        else
  171
+          $stderr.puts(message)
  172
+        end
  173
+        nil
  174
+      end
  175
+
158 176
       protected
159 177
 
160 178
       def new_request
@@ -204,13 +222,30 @@ def async_request_with_retries response, http_request, retry_delays = nil
204 222
 
205 223
       end
206 224
 
207  
-      def make_sync_request response
  225
+      def make_sync_request response, &read_block
208 226
         retry_server_errors do
209 227
 
210  
-          response.http_response = http_response =
211  
-            Http::Response.new
  228
+          response.http_response = Http::Response.new
  229
+
  230
+          @http_handler.handle(
  231
+            response.http_request,
  232
+            response.http_response,
  233
+            &read_block)
  234
+
  235
+          if
  236
+            block_given? and
  237
+            response.http_response.status < 300 and
  238
+            response.http_response.body
  239
+          then
  240
+
  241
+            msg = ":http_handler read the entire http response body into "
  242
+            msg << "memory, it should have instead yielded chunks"
  243
+            log_warning(msg)
  244
+
  245
+            # go ahead and yield the body on behalf of the handler
  246
+            yield(response.http_response.body)
212 247
 
213  
-          @http_handler.handle(response.http_request, http_response)
  248
+          end
214 249
 
215 250
           populate_error(response)
216 251
           response.signal_success unless response.error
@@ -227,7 +262,6 @@ def retry_server_errors &block
227 262
         while should_retry?(response)
228 263
           break if sleeps.empty?
229 264
           Kernel.sleep(sleeps.shift)
230  
-          # rebuild the request to get a fresh signature
231 265
           rebuild_http_request(response)
232 266
           response = yield
233 267
         end
@@ -256,8 +290,16 @@ def scaling_factor response
256 290
       end
257 291
 
258 292
       def should_retry? response
  293
+        if retryable_error?(response)
  294
+          response.safe_to_retry?
  295
+        else
  296
+          false
  297
+        end
  298
+      end
  299
+
  300
+      def retryable_error? response
259 301
         expired_credentials?(response) or
260  
-        response.timeout? or
  302
+        response.network_error? or
261 303
         response.throttled? or
262 304
         response.error.kind_of?(Errors::ServerError)
263 305
       end
@@ -303,7 +345,7 @@ def log_client_request options, &block
303 345
       end
304 346
 
305 347
       # Logs the response to the configured logger.
306  
-      # @param [Resposne] response
  348
+      # @param [Response] response
307 349
       # @return [nil]
308 350
       def log_response response
309 351
         if config.logger
@@ -336,10 +378,10 @@ def extract_error response
336 378
         ]
337 379
 
338 380
         case
339  
-        when response.timeout? then TimeoutError.new
340  
-        when error_code        then error_class(error_code).new(*error_args)
341  
-        when status >= 500     then Errors::ServerError.new(*error_args)
342  
-        when status >= 300     then Errors::ClientError.new(*error_args)
  381
+        when response.network_error? then NetworkError.new
  382
+        when error_code              then error_class(error_code).new(*error_args)
  383
+        when status >= 500           then Errors::ServerError.new(*error_args)
  384
+        when status >= 300           then Errors::ClientError.new(*error_args)
343 385
         else nil # no error
344 386
         end
345 387
 
@@ -371,21 +413,25 @@ def errors_module
371 413
         AWS.const_get(self.class.to_s[/(\w+)::Client/, 1])::Errors
372 414
       end
373 415
 
374  
-      def client_request name, options, &block
  416
+      def client_request name, options, &read_block
375 417
         return_or_raise(options) do
376 418
           log_client_request(options) do
377 419
 
378 420
             if config.stub_requests?
379 421
 
380 422
               response = stub_for(name)
381  
-              response.http_request = build_request(name, options, &block)
  423
+              response.http_request = build_request(name, options)
382 424
               response.request_options = options
383 425
               response
384 426
 
385 427
             else
386 428
 
387 429
               client = self
388  
-              response = new_response { client.send(:build_request, name, options, &block) }
  430
+
  431
+              response = new_response do
  432
+                client.send(:build_request, name, options)
  433
+              end
  434
+
389 435
               response.request_type = name
390 436
               response.request_options = options
391 437
 
@@ -399,8 +445,8 @@ def client_request name, options, &block
399 445
               else
400 446
                 # process the http request
401 447
                 options[:async] ?
402  
-                make_async_request(response) :
403  
-                  make_sync_request(response)
  448
+                make_async_request(response, &read_block) :
  449
+                  make_sync_request(response, &read_block)
404 450
 
405 451
                 # process the http response
406 452
                 response.on_success do
@@ -424,7 +470,7 @@ def cacheable_request? name, options
424 470
         self.class::CACHEABLE_REQUESTS.include?(name)
425 471
       end
426 472
 
427  
-      def build_request(name, options, &block)
  473
+      def build_request name, options
428 474
 
429 475
         # we dont want to pass the async option to the configure block
430 476
         opts = options.dup
@@ -445,7 +491,7 @@ def build_request(name, options, &block)
445 491
         http_request.ssl_ca_file = config.ssl_ca_file if config.ssl_ca_file
446 492
         http_request.ssl_ca_path = config.ssl_ca_path if config.ssl_ca_path
447 493
 
448  
-        send("configure_#{name}_request", http_request, opts, &block)
  494
+        send("configure_#{name}_request", http_request, opts)
449 495
 
450 496
         http_request.headers["user-agent"] = user_agent_string
451 497
         http_request.add_authorization!(credential_provider)
@@ -474,16 +520,18 @@ def user_agent_string
474 520
       #
475 521
       def self.add_client_request_method method_name, options = {}, &block
476 522
 
477  
-        self.operations << method_name
  523
+        operations << method_name
478 524
 
479 525
         ClientRequestMethodBuilder.new(self, method_name, &block)
480 526
 
481  
-        module_eval <<-END
  527
+        method_def = <<-METHOD
482 528
           def #{method_name}(*args, &block)
483 529
             options = args.first ? args.first : {}
484 530
             client_request(#{method_name.inspect}, options, &block)
485 531
           end
486  
-        END
  532
+        METHOD
  533
+
  534
+        module_eval(method_def)
487 535
 
488 536
       end
489 537
 
@@ -518,15 +566,6 @@ def initialize client_class, method_name, &block
518 566
         def configure_request options = {}, &block
519 567
           name = "configure_#{@method_name}_request"
520 568
           MetaUtils.class_extend_method(@client_class, name, &block)
521  
-          if block.arity == 3
522  
-            m = Module.new
523  
-            m.module_eval(<<-END)
524  
-              def #{name}(req, options, &block)
525  
-                super(req, options, block)
526  
-              end
527  
-            END
528  
-            @client_class.send(:include, m)
529  
-          end
530 569
         end
531 570
 
532 571
         def process_response &block
13  lib/aws/core/configuration.rb
@@ -128,7 +128,7 @@ module Core
128 128
     #   +true+, requests will always use path style.  This can be useful
129 129
     #   for testing environments.
130 130
     #
131  
-    # @attr_reader [Integer] s3_multipart_max_parts (1000)
  131
+    # @attr_reader [Integer] s3_multipart_max_parts (10000)
132 132
     #   The maximum number of parts to split a file into when uploading
133 133
     #   in parts to S3.
134 134
     #
@@ -163,6 +163,17 @@ module Core
163 163
     #
164 164
     #     s3 = AWS::S3.new(:s3_server_side_encryption => :aes256)
165 165
     #
  166
+    # @attr_reader [OpenSSL::PKey::RSA, String] s3_encryption_key
  167
+    #   If this is set, AWS::S3::S3Object #read and #write methods will always
  168
+    #   perform client-side encryption with this key. The key can be overridden
  169
+    #   at runtime by using the :encryption_key option.  A value of nil
  170
+    #   means that client-side encryption will not be used.
  171
+    #
  172
+    # @attr_reader [Symbol] s3_encryption_materials_location
  173
+    #   When set to +:instruction_file+, AWS::S3::S3Object will store
  174
+    #   encryption materials in a seperate object, instead of the object
  175
+    #   metadata.
  176
+    #
166 177
     # @attr_reader [String] simple_db_endpoint ('sdb.amazonaws.com')
167 178
     #   The service endpoint for Amazon SimpleDB.
168 179
     #
44  lib/aws/core/http/handler.rb
@@ -14,27 +14,39 @@
14 14
 module AWS
15 15
   module Core
16 16
     module Http
17  
-      
  17
+
18 18
       # @private
19 19
       class Handler
20  
-  
  20
+
21 21
         attr_reader :base
22  
-  
  22
+
23 23
         def initialize(base, &block)
24 24
           @base = base
25 25
           if base.respond_to?(:handle)
26  
-  
27  
-            unless block.arity == 2
28  
-              raise ArgumentError, 'passed block must accept 2 arguments'
  26
+
  27
+            unless [2,3].include?(block.arity)
  28
+              raise ArgumentError, 'passed block must accept 2 or 3 arguments'
29 29
             end
  30
+
30 31
             MetaUtils.extend_method(self, :handle, &block)
31  
-  
  32
+
  33
+            if block.arity == 3
  34
+              m = Module.new do
  35
+                eval(<<-DEF)
  36
+                  def handle req, resp, &read_block
  37
+                    super(req, resp, read_block)
  38
+                  end
  39
+                DEF
  40
+              end
  41
+              self.extend(m)
  42
+            end
  43
+
32 44
           elsif base.respond_to?(:handle_async)
33  
-  
  45
+
34 46
             unless block.arity == 3
35 47
               raise ArgumentError, 'passed block must accept 3 arguments'
36 48
             end
37  
-