Support S3 IAD Regional option (#2194)

* Support S3 IAD Regional option

* Undo 2.11.419 bump

* Add downcase for regional option

CHANGELOG.md

@@ -1,8 +1,7 @@
 Unreleased Changes
 ------------------
 
-2.11.419 (2019-12-18)
-------------------
+* Feature - Support `s3_us_east_1_regional_endpoint` from `SharedConfig`.
 
 * Feature - Aws::CloudFront - Updated the documentation for Amazon CloudFront.
 
@@ -1444,7 +1443,7 @@ Unreleased Changes
 
 * Feature - Aws::IoTEvents - Added support for a new service, AWS IoT Events.
 
-* Issue - Improved exception messages in credential providers to exclude detailed parse errors that may contain sensitive information. 
+* Issue - Improved exception messages in credential providers to exclude detailed parse errors that may contain sensitive information.
 
 2.11.284 (2019-05-29)
 ------------------

VERSION

@@ -1 +1 @@
-2.11.419
+2.11.418

aws-sdk-core/lib/aws-sdk-core.rb

@@ -348,6 +348,7 @@ module Plugins
     autoload :S3GetBucketLocationFix, 'aws-sdk-core/plugins/s3_get_bucket_location_fix'
     autoload :S3Http200Errors, 'aws-sdk-core/plugins/s3_http_200_errors'
     autoload :S3HostId, 'aws-sdk-core/plugins/s3_host_id'
+    autoload :S3IADRegionalEndpoint, 'aws-sdk-core/plugins/s3_iad_regional_endpoint'
     autoload :S3LocationConstraint, 'aws-sdk-core/plugins/s3_location_constraint'
     autoload :S3Md5s, 'aws-sdk-core/plugins/s3_md5s'
     autoload :S3Redirects, 'aws-sdk-core/plugins/s3_redirects'

aws-sdk-core/lib/aws-sdk-core/api/customizations.rb

@@ -265,6 +265,7 @@ def is_eventstream?(api, shape_name)
         Aws::Plugins::S3Expect100Continue
         Aws::Plugins::S3HostId
         Aws::Plugins::S3Http200Errors
+        Aws::Plugins::S3IADRegionalEndpoint
         Aws::Plugins::S3GetBucketLocationFix
         Aws::Plugins::S3LocationConstraint
         Aws::Plugins::S3Md5s

aws-sdk-core/lib/aws-sdk-core/endpoint_provider.rb

@@ -28,7 +28,7 @@ def initialize(rules)
       @rules = rules
     end
 
-    def resolve(region, service, sts_regional_endpoints = nil)
+    def resolve(region, service, sts_regional_endpoints)
       "https://" + endpoint_for(region, service, sts_regional_endpoints)
     end
 
@@ -49,7 +49,7 @@ def dns_suffix_for(region)
 
     private
 
-    def endpoint_for(region, service, sts_regional_endpoints)
+    def endpoint_for(region, service, sts_regional_endpoints = 'legacy')
       partition = get_partition(region)
       endpoint = default_endpoint(partition, service, region)
       service_cfg = partition.fetch("services", {}).fetch(service, {})

aws-sdk-core/lib/aws-sdk-core/plugins/regional_endpoint.rb

@@ -25,8 +25,9 @@ class RegionalEndpoint < Seahorse::Client::Plugin
       option(:endpoint) do |cfg|
         endpoint_prefix = cfg.api.metadata['endpointPrefix']
         if cfg.region && endpoint_prefix
-          EndpointProvider.resolve(cfg.region, endpoint_prefix)
-          sts_regional = cfg.respond_to?(:sts_regional_endpoints) ? cfg.sts_regional_endpoints : nil
+          if cfg.respond_to?(:sts_regional_endpoints)
+            sts_regional = cfg.sts_regional_endpoints
+          end
           EndpointProvider.resolve(cfg.region, endpoint_prefix, sts_regional)
         end
       end

aws-sdk-core/lib/aws-sdk-core/plugins/s3_iad_regional_endpoint.rb

@@ -0,0 +1,58 @@
+module Aws
+  module Plugins
+
+    class S3IADRegionalEndpoint < Seahorse::Client::Plugin
+
+      option(:s3_us_east_1_regional_endpoint,
+        default: 'legacy',
+        doc_type: String,
+        docstring: <<-DOCS) do |cfg|
+Passing in `regional` to enable regional endpoint for S3's `us-east-1`
+region. Defaults to `legacy` mode using global endpoint.
+        DOCS
+        resolve_iad_regional_endpoint(cfg)
+      end
+
+      def add_handlers(handlers, config)
+        if config.region == 'us-east-1'
+          handlers.add(Handler)
+        end
+      end
+
+      # @api private
+      class Handler < Seahorse::Client::Handler
+
+        def call(context)
+          # keep legacy global endpoint pattern by default
+          if context.config.s3_us_east_1_regional_endpoint == 'legacy'
+            context.http_request.endpoint.host = S3IADRegionalEndpoint.legacy_host(
+              context.http_request.endpoint.host)
+          end
+          @handler.call(context)
+        end
+
+      end
+
+      def self.legacy_host(host)
+        host.sub(".us-east-1", '')
+      end
+
+      private
+
+      def self.resolve_iad_regional_endpoint(cfg)
+        mode = ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] ||
+          Aws.shared_config.s3_us_east_1_regional_endpoint(profile: cfg.profile) ||
+          'legacy'
+        mode.downcase!
+        unless %w(legacy regional).include?(mode)
+          raise ArgumentError, "expected :s3_us_east_1_regional_endpoint or"\
+            " ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] to be `legacy` or"\
+            " `regional`."
+        end
+        mode
+      end
+
+    end
+
+  end
+end

aws-sdk-core/lib/aws-sdk-core/shared_config.rb

@@ -143,6 +143,21 @@ def sts_regional_endpoints(opts = {})
       end
     end
 
+    def s3_us_east_1_regional_endpoint(opts = {})
+      p = opts[:profile] || @profile_name
+      if @config_enabled
+        if @parsed_credentials
+          mode = @parsed_credentials.fetch(p, {})["s3_us_east_1_regional_endpoint"]
+        end
+        if @parsed_config
+          mode ||= @parsed_config.fetch(p, {})["s3_us_east_1_regional_endpoint"]
+        end
+        mode
+      else
+        nil
+      end
+    end
+
     def endpoint_discovery(opts = {})
       p = opts[:profile] || @profile_name
       if @config_enabled && @parsed_config

aws-sdk-core/lib/aws-sdk-core/version.rb

@@ -1,3 +1,3 @@
 module Aws
-  VERSION = '2.11.419'
+  VERSION = '2.11.418'
 end

aws-sdk-core/spec/aws/s3/client/iad_regional_endpoint_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+module Aws
+  module S3
+    describe Client do
+
+      describe ':s3_us_east_1_regional_endpoint' do
+
+        it 'defaults to `legacy`' do
+          client = Client.new(stub_responses: true, region: 'us-east-1')
+          expect(client.config.s3_us_east_1_regional_endpoint).to eq('legacy')
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.amazonaws.com/')
+        end
+
+        it 'can be set to `regional` in the constructor' do
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-east-1',
+            s3_us_east_1_regional_endpoint: 'regional'
+          )
+          expect(client.config.s3_us_east_1_regional_endpoint).to eq('regional')
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.us-east-1.amazonaws.com/')
+        end
+
+        it 'raise an error when invalid option provided' do
+          ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] = 'foo'
+          expect do
+            Client.new(
+              stub_responses: true,
+              region: 'us-east-1'
+            )
+          end.to raise_error(ArgumentError)
+        end
+
+        it 'can be set from ENV' do
+          ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] = 'regional'
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-east-1'
+          )
+          expect(client.config.s3_us_east_1_regional_endpoint).to eq('regional')
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.us-east-1.amazonaws.com/')
+        end
+
+        it 'uses client option before ENV' do
+          ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] = 'legacy'
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-east-1',
+            s3_us_east_1_regional_endpoint: 'regional'
+          )
+          expect(client.config.s3_us_east_1_regional_endpoint).to eq('regional')
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.us-east-1.amazonaws.com/')
+        end
+
+        it 'is case insensitive' do
+          ENV['AWS_S3_US_EAST_1_REGIONAL_ENDPOINT'] = 'LEGACY'
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-east-1'
+          )
+          expect(client.config.s3_us_east_1_regional_endpoint).to eq('legacy')
+        end
+
+        it 'has no effect on non-IAD regions' do
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-west-2',
+            s3_us_east_1_regional_endpoint: 'legacy'
+          )
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.us-west-2.amazonaws.com/')
+
+          client = Client.new(
+            stub_responses: true,
+            region: 'us-west-2',
+            s3_us_east_1_regional_endpoint: 'regional'
+          )
+          resp = client.list_buckets
+          expect(resp.context.http_request.endpoint.to_s).to eq(
+            'https://s3.us-west-2.amazonaws.com/')
+        end
+
+      end
+
+    end
+  end
+end

aws-sdk-core/spec/aws/shared_config_spec.rb

@@ -130,6 +130,26 @@ module Aws
 
       end
 
+      context 's3_us_east_1_regional_endpoint selection' do
+
+        it 'can resolve s3_us_east_1_regional_endpoint from config file' do
+          config = SharedConfig.new(
+            config_path: mock_config_file,
+            config_enabled: true,
+            profile_name: "s3_iad_regional"
+          )
+          expect(config.s3_us_east_1_regional_endpoint).to eq('regional')
+
+          config = SharedConfig.new(
+            config_path: mock_config_file,
+            config_enabled: true,
+            profile_name: "s3_iad_legacy"
+          )
+          expect(config.s3_us_east_1_regional_endpoint).to eq('legacy')
+        end
+
+      end
+
     end
   end
 end

aws-sdk-core/spec/fixtures/credentials/mock_shared_config

@@ -88,3 +88,13 @@ sts_regional_endpoints = regional
 aws_access_key_id = AKID
 aws_secret_access_key = SECRET
 sts_regional_endpoints = legacy
+
+[profile s3_iad_regional]
+aws_access_key_id = AKID
+aws_secret_access_key = SECRET
+s3_us_east_1_regional_endpoint = regional
+
+[profile s3_iad_legacy]
+aws_access_key_id = AKID
+aws_secret_access_key = SECRET
+s3_us_east_1_regional_endpoint = legacy

aws-sdk-resources/lib/aws-sdk-resources/services/s3/bucket.rb

@@ -114,6 +114,10 @@ def s3_bucket_url
           url.path += '/' unless url.path[-1] == '/'
           url.path += Seahorse::Util.uri_escape(name)
         end
+        if (client.config.region == 'us-east-1') &&
+          (client.config.s3_us_east_1_regional_endpoint == 'legacy')
+          url.host = Plugins::S3IADRegionalEndpoint.legacy_host(url.host)
+        end
         url.to_s
       end
 

aws-sdk-resources/lib/aws-sdk-resources/services/s3/presigned_post.rb

@@ -585,6 +585,10 @@ def bucket_url
         else
           url.path = '/' + @bucket_name
         end
+        if @bucket_region == 'us-east-1'
+          # keep legacy behavior by default
+          url.host = Plugins::S3IADRegionalEndpoint.legacy_host(url.host)
+        end
         url.to_s
       end