Flexible Checksums Pt2 (#2668)

gems/aws-sdk-s3/lib/aws-sdk-s3/multipart_file_uploader.rb

@@ -21,6 +21,10 @@ class MultipartFileUploader
         Client.api.operation(:create_multipart_upload).input.shape.member_names
       )
 
+      COMPLETE_OPTIONS = Set.new(
+        Client.api.operation(:complete_multipart_upload).input.shape.member_names
+      )
+
       # @api private
       UPLOAD_PART_OPTIONS = Set.new(
         Client.api.operation(:upload_part).input.shape.member_names
@@ -42,7 +46,7 @@ def initialize(options = {})
       # @option options [Proc] :progress_callback
       #   A Proc that will be called when each chunk of the upload is sent.
       #   It will be invoked with [bytes_read], [total_sizes]
-      # @return [void]
+      # @return [Seahorse::Client::Response] - the CompleteMultipartUploadResponse
       def upload(source, options = {})
         if File.size(source) < MIN_PART_SIZE
           raise ArgumentError, FILE_TOO_SMALL
@@ -61,10 +65,10 @@ def initiate_upload(options)
 
       def complete_upload(upload_id, parts, options)
         @client.complete_multipart_upload(
-          bucket: options[:bucket],
-          key: options[:key],
-          upload_id: upload_id,
-          multipart_upload: { parts: parts }
+          **complete_opts(options).merge(
+            upload_id: upload_id,
+            multipart_upload: { parts: parts }
+          )
         )
       end
 
@@ -123,6 +127,13 @@ def create_opts(options)
         end
       end
 
+      def complete_opts(options)
+        COMPLETE_OPTIONS.inject({}) do |hash, key|
+          hash[key] = options[key] if options.key?(key)
+          hash
+        end
+      end
+
       def upload_part_opts(options)
         UPLOAD_PART_OPTIONS.inject({}) do |hash, key|
           hash[key] = options[key] if options.key?(key)
@@ -147,7 +158,15 @@ def upload_in_threads(pending, completed, options)
                 end
                 resp = @client.upload_part(part)
                 part[:body].close
-                completed.push(etag: resp.etag, part_number: part[:part_number])
+                completed_part = {etag: resp.etag, part_number: part[:part_number]}
+
+                # get the requested checksum from the response
+                if part[:checksum_algorithm]
+                  k = "checksum_#{part[:checksum_algorithm].downcase}".to_sym
+                  completed_part[k] = resp[k]
+                end
+
+                completed.push(completed_part)
               end
               nil
             rescue => error
@@ -224,4 +243,4 @@ def call(part_number, bytes_read)
       end
     end
   end
-end
+end

gems/aws-sdk-s3/spec/plugins/skip_whole_multipart_checksums_spec.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require_relative '../spec_helper'
+
+module Aws
+  module S3
+    module Plugins
+      describe SkipWholeMultipartGetChecksums do
+        let(:creds) { Aws::Credentials.new('akid', 'secret') }
+        let(:client) { S3::Client.new(stub_responses: true) }
+        let(:bucket) { 'bucket' }
+        let(:key) { 'key' }
+
+        let(:body) { 'hello world' }
+        let(:digest) { 'DUoRhQ==' }
+
+        let(:body_part_1) { 'hello '}
+        let(:digest_part_1) { '7YH59g==' }
+
+        it 'validates the checksum on an Object GET' do
+          client.stub_responses(
+            :get_object,
+            [{
+               body: body,
+               headers: {'x-amz-checksum-crc32' => digest},
+               status_code: 200
+             }])
+          resp = client.get_object(bucket: bucket, key: key, checksum_mode: 'ENABLED')
+          expect(resp.context[:http_checksum][:validated]).to eq 'CRC32'
+        end
+
+        it 'raises when the checksum does not match on an Object GET' do
+          client.stub_responses(
+            :get_object,
+            [{
+               body: body,
+               headers: {'x-amz-checksum-crc32' => 'invalid_value'},
+               status_code: 200
+             }])
+          expect do
+            client.get_object(bucket: bucket, key: key, checksum_mode: 'ENABLED')
+          end.to raise_error(Aws::Errors::ChecksumError)
+        end
+
+        it 'validates the checksum on a range GET matching the part boundary' do
+          client.stub_responses(
+            :get_object,
+            [{
+               body: body_part_1,
+               headers: {'x-amz-checksum-crc32' => digest_part_1},
+               status_code: 200
+             }])
+          resp = client.get_object(bucket: bucket, key: key, checksum_mode: 'ENABLED', range: "bytes=0-6")
+          expect(resp.context[:http_checksum][:validated]).to eq 'CRC32'
+        end
+
+        it 'validates the checksum on a single part GET' do
+          client.stub_responses(
+            :get_object,
+            [{
+               body: body_part_1,
+               headers: {'x-amz-checksum-crc32' => digest_part_1},
+               status_code: 200
+             }])
+          resp = client.get_object(bucket: bucket, key: key, checksum_mode: 'ENABLED', part_number: 1)
+          expect(resp.context[:http_checksum][:validated]).to eq 'CRC32'
+        end
+
+        it 'does not validate on a whole multipart GET' do
+          client.stub_responses(
+            :get_object,
+            [{
+               body: body,
+               headers: {'x-amz-checksum-crc32' => 'cpjwid==-12'},
+               status_code: 200
+             }])
+          resp = client.get_object(bucket: bucket, key: key, checksum_mode: 'ENABLED')
+          expect(resp.context[:http_checksum][:validated]).to be_nil
+        end
+      end
+    end
+  end
+end