AWS SDK Compatibility: Check for http_proxy & https_proxy environ…

…ment variables (#2991)

* check for `http_proxy` & `https_proxy` env-vars

this brings us in line with many other AWS SDKs which check for
these environment variables to modify client behavior. it does not
change behavior if you've already been setting your web proxy
explicitly, or in properties (the same as the java (and other) sdks).

we do introduce this with two new methods in ClientConfig to not break
any existing callers using the methods that are public. while all of our
callers have been updated to properly check web proxy, and then if
that's unset check the protocol being used and optionally call
`GetHttpProxy`/`GetHttpsProxy` as needed. If we're okay with breaking
ever IT MAY be worthwhile to break `GetWebProxy` by taking in the
protocol, and then just having the one method to prevent any accidental
mis-use, and to ensure everyone is using those environment variables.

it's also important to note:
<dotnet/runtime#31113> meaning a user will
just get an error if they try to proxy to something that is using
`HTTPS`. i figured just letting it error on construction is probably
the 'safest' option. letting the user know that their setting isn't
being respected, as opposed to just silently 'ignoring' it. this is
also a decision we might want to change.

* actually parse out credentials for proxy

* remove accidental .swp file

* swap reading explicit over env-var first, tests

Ensure we read explicitly configured proxies through code first,
and then fallback to reading the proxy from environment variables.
next add some tests with the recommended environment variable mocking
test setup provided.

* address comments about BCL ClientConfig

sdk/src/Core/Amazon.Runtime/ClientConfig.cs

@@ -128,6 +128,33 @@ private CredentialProfileStoreChain CredentialProfileStoreChain
             }
         }
 
+#if BCL
+        private static WebProxy GetWebProxyWithCredentials(string value)
+#else
+        private static Amazon.Runtime.Internal.Util.WebProxy? GetWebProxyWithCredentials(string value)
+#endif
+        {
+            if (!string.IsNullOrEmpty(value))
+            {
+                var asUri = new Uri(value);
+#if BCL
+                var parsedProxy = new WebProxy(asUri);
+#else
+                var parsedProxy = new Amazon.Runtime.Internal.Util.WebProxy(asUri);
+#endif
+                if (!string.IsNullOrEmpty(asUri.UserInfo)) {
+                    var userAndPass = asUri.UserInfo.Split(':');
+                    parsedProxy.Credentials = new NetworkCredential(
+                        userAndPass[0],
+                        userAndPass.Length > 1 ? userAndPass[1] : string.Empty
+                    );
+                }
+                return parsedProxy;
+            }
+
+            return null;
+        }
+
         /// <inheritdoc />
         public IAWSTokenProvider AWSTokenProvider
         {
@@ -647,6 +674,34 @@ public ICredentials ProxyCredentials
             set { this.proxyCredentials = value; }
         }
 
+#if BCL
+        public WebProxy GetHttpProxy()
+#else
+        public IWebProxy GetHttpProxy()
+#endif
+        {
+            var explicitProxy = GetWebProxy();
+            if (explicitProxy != null)
+            {
+                return explicitProxy;
+            }
+            return GetWebProxyWithCredentials(Environment.GetEnvironmentVariable("http_proxy"));
+        }
+
+#if BCL
+        public WebProxy GetHttpsProxy()
+#else
+        public IWebProxy GetHttpsProxy()
+#endif
+        {
+            var explicitProxy = GetWebProxy();
+            if (explicitProxy != null)
+            {
+                return explicitProxy;
+            }
+            return GetWebProxyWithCredentials(Environment.GetEnvironmentVariable("https_proxy"));
+        }
+
 #if BCL
         /// <summary>
         /// Specifies the TCP keep-alive values to use for service requests.
@@ -1175,4 +1230,4 @@ public int HttpClientCacheSize
         /// </summary>
         public IEndpointProvider EndpointProvider { get; set; }
     }
-}
+}

sdk/src/Core/Amazon.Runtime/Pipeline/HttpHandler/_bcl/HttpWebRequestFactory.cs

@@ -474,6 +474,14 @@ public virtual void ConfigureRequest(IRequestContext requestContext)
                 requestContext.Metrics.AddProperty(Metric.ProxyPort, requestContext.ClientConfig.ProxyPort);
                 _request.Proxy = proxy;
             }
+            else if (_request.RequestUri.Scheme == Uri.UriSchemeHttp)
+            {
+                _request.Proxy = requestContext.ClientConfig.GetHttpProxy();
+            }
+            else if (_request.RequestUri.Scheme == Uri.UriSchemeHttps)
+            {
+                _request.Proxy = requestContext.ClientConfig.GetHttpsProxy();
+            }
 
             // Set service point properties.
             _request.ServicePoint.ConnectionLimit = clientConfig.ConnectionLimit;

sdk/src/Core/Amazon.Runtime/Pipeline/HttpHandler/_netstandard/HttpRequestMessageFactory.cs

@@ -270,7 +270,11 @@ private static HttpClient CreateManagedHttpClient(IClientConfig clientConfig)
             }
 
             try
-            { 
+            {
+                // HTTP Client will automatically read `HTTP_PROXY`,
+                // and `HTTPS_PROXY`. So we let it use those variables, since
+                // we don't know at client construction time whether they're
+                // using HTTP or HTTPS.
                 var proxy = clientConfig.GetWebProxy();
                 if (proxy != null)
                 {

sdk/src/Core/Amazon.Runtime/_bcl/IClientConfig.bcl.cs

@@ -70,5 +70,16 @@ public partial interface IClientConfig
         /// </summary>
         WebProxy GetWebProxy();
 
+        /// <summary>
+        /// Returns a WebProxy instance to use for HTTPS connections if an
+        /// explicit web proxy hasn't been configured.
+        /// </summary>
+        WebProxy GetHttpsProxy();
+
+        /// <summary>
+        /// Returns a WebProxy instance to use for HTTP connections if an
+        /// explicit web proxy hasn't been configured.
+        /// </summary>
+        WebProxy GetHttpProxy();
     }
 }

sdk/src/Core/Amazon.Runtime/_netstandard/IClientConfig.cs

@@ -43,6 +43,18 @@ public partial interface IClientConfig
         /// </summary>
         IWebProxy GetWebProxy();
 
+        /// <summary>
+        /// Returns a WebProxy instance to use for HTTPS connections if an
+        /// explicit web proxy hasn't been configured.
+        /// </summary>
+        IWebProxy GetHttpsProxy();
+
+        /// <summary>
+        /// Returns a WebProxy instance to use for HTTP connections if an
+        /// explicit web proxy hasn't been configured.
+        /// </summary>
+        IWebProxy GetHttpProxy();
+
         /// <summary>
         /// HttpClientFactory used to create new HttpClients.
         /// If null, an HttpClient will be created by the SDK.

sdk/src/Services/EC2/Custom/Util/_async/ImageUtilities.async.cs

@@ -82,10 +82,17 @@ private static async Task LoadDefinitionsFromWebAsync(AmazonEC2Config ec2Config)
                 if (ImageDefinitionsLoaded)
                     return;
             }
+            const string httpPrefix = "http://";
+            const string httpsPrefix = "https://";
 
             IWebProxy webProxy = null;
-            if (ec2Config != null)
+            IWebProxy httpProxy = null;
+            IWebProxy httpsProxy = null;
+            if (ec2Config != null) {
                 webProxy = ec2Config.GetWebProxy();
+                httpProxy = ec2Config.GetHttpProxy();
+                httpsProxy = ec2Config.GetHttpsProxy();
+            }
 
             int retries = 0;
             while (retries < MAX_DOWNLOAD_RETRIES)
@@ -95,9 +102,22 @@ private static async Task LoadDefinitionsFromWebAsync(AmazonEC2Config ec2Config)
                     HttpWebResponse response = null;
                     foreach (var location in DownloadLocations)
                     {
+                        var useProxy = webProxy;
+                        if (useProxy == null)
+                        {
+                            if (location.StartsWith(httpPrefix))
+                            {
+                                useProxy = httpProxy;
+                            }
+                            else if (location.StartsWith(httpsPrefix))
+                            {
+                                useProxy = httpsProxy;
+                            }
+                        }
+
                         try
                         {
-                            response = await DownloadControlFileAsync(location, webProxy).ConfigureAwait(false);
+                            response = await DownloadControlFileAsync(location, useProxy).ConfigureAwait(false);
                             if (response != null)
                                 break;
                         }

sdk/src/Services/EC2/Custom/Util/_bcl/ImageUtilities.bcl.cs

@@ -82,10 +82,17 @@ private static void LoadDefinitionsFromWeb(AmazonEC2Config ec2Config)
                 if (ImageDefinitionsLoaded)
                     return;
             }
+            const string httpPrefix = "http://";
+            const string httpsPrefix = "https://";
 
             IWebProxy webProxy = null;
-            if (ec2Config != null)
+            IWebProxy httpProxy = null;
+            IWebProxy httpsProxy = null;
+            if (ec2Config != null) {
                 webProxy = ec2Config.GetWebProxy();
+                httpProxy = ec2Config.GetHttpProxy();
+                httpsProxy = ec2Config.GetHttpsProxy();
+            }
 
             int retries = 0;
             while (retries < MAX_DOWNLOAD_RETRIES)
@@ -95,9 +102,22 @@ private static void LoadDefinitionsFromWeb(AmazonEC2Config ec2Config)
                     HttpWebResponse response = null;
                     foreach (var location in DownloadLocations)
                     {
+                        var useProxy = webProxy;
+                        if (useProxy == null)
+                        {
+                            if (location.StartsWith(httpPrefix))
+                            {
+                                useProxy = httpProxy;
+                            }
+                            else if (location.StartsWith(httpsPrefix))
+                            {
+                                useProxy = httpsProxy;
+                            }
+                        }
+
                         try
                         {
-                            response = DownloadControlFile(location, webProxy);
+                            response = DownloadControlFile(location, useProxy);
                             if (response != null)
                                 break;
                         }

sdk/src/Services/S3/Custom/AmazonS3Client.Extensions.cs

@@ -482,6 +482,17 @@ internal void ConfigureProxy(HttpWebRequest httpRequest)
             {
                 httpRequest.Proxy.Credentials = Config.ProxyCredentials;
             }
+            if (httpRequest.Proxy == null)
+            {
+                if (httpRequest.RequestUri.Scheme == Uri.UriSchemeHttp)
+                {
+                    httpRequest.Proxy = Config.GetHttpProxy();
+                }
+                else if (httpRequest.RequestUri.Scheme == Uri.UriSchemeHttps)
+                {
+                    httpRequest.Proxy = Config.GetHttpsProxy();
+                }
+            }
         }
 
 

sdk/src/Services/S3/Custom/Util/AmazonS3HttpUtil.cs

@@ -112,15 +112,19 @@ private static GetHeadResponse HandleWebException(string header, WebException we
 
         private static void SetProxyIfAvailableAndConfigured(IClientConfig config, HttpWebRequest httpWebRequest)
         {
-            var proxy = GetProxyIfAvailableAndConfigured(config);
+            var proxy = config.GetWebProxy();
             if (proxy != null)
             {
                 httpWebRequest.Proxy = proxy;
             }
-        }
-        private static IWebProxy GetProxyIfAvailableAndConfigured(IClientConfig config)
-        {
-            return config.GetWebProxy();
+            else if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttp)
+            {
+                httpWebRequest.Proxy = config.GetHttpProxy();
+            }
+            else if (httpWebRequest.RequestUri.Scheme == Uri.UriSchemeHttps)
+            {
+                httpWebRequest.Proxy = config.GetHttpsProxy();
+            }
         }
     }
 }

sdk/src/Services/SecurityToken/Custom/AmazonSecurityTokenServiceClient.Extension.cs

@@ -40,11 +40,25 @@ public partial class AmazonSecurityTokenServiceClient : AmazonServiceClient, IAm
             TimeSpan credentialDuration,
             ICredentials userCredential)
         {
+            const string httpPrefix = "http://";
+            const string httpsPrefix = "https://";
             SAMLAssertion assertion;
 
             try
             {
-                var authController = new SAMLAuthenticationController(Config.GetWebProxy());
+                var proxy = Config.GetWebProxy();
+                if (proxy == null)
+                {
+                    if (endpoint.StartsWith(httpPrefix))
+                    {
+                        proxy = Config.GetHttpProxy();
+                    }
+                    else if (endpoint.StartsWith(httpsPrefix))
+                    {
+                        proxy = Config.GetHttpsProxy();
+                    }
+                }
+                var authController = new SAMLAuthenticationController(proxy);
                 assertion = authController.GetSAMLAssertion(endpoint, userCredential, authenticationType);
             }
             catch (Exception e)

sdk/test/UnitTests/Custom/Runtime/ProxyTests.cs

@@ -23,6 +23,7 @@ namespace AWSSDK.UnitTests
     [TestClass()]
     public class ProxyTests
     {
+        readonly string EnvironmentVariableUrl = "http://user:pass@10.0.0.2:21/proxy";
         readonly string Host = "10.0.0.1";
         readonly int Port = 20;
         readonly List<string> BypassList = new List<string>
@@ -92,6 +93,65 @@ public void TestProxySetupWithSchemedHost()
             WebProxy proxy = dummyConfig.GetWebProxy();
             Assert.IsTrue(proxy.Address.ToString().StartsWith(host, StringComparison.OrdinalIgnoreCase));
         }
+
+        [TestMethod]
+        [TestCategory("UnitTest")]
+        [TestCategory("Runtime")]
+        public void TestParsingCredentialsOverEnvironmentVariable()
+        {
+            var cachedHttpProxy = Environment.GetEnvironmentVariable("http_proxy");
+
+            try
+            {
+                Environment.SetEnvironmentVariable("http_proxy", EnvironmentVariableUrl);
+
+                var dummyConfig = new AmazonEC2Config();
+                IWebProxy proxy = dummyConfig.GetHttpProxy();
+
+                var address = proxy.GetProxy(new Uri("https://ec2.us-west-2.aws.amazon.com"));
+                Assert.AreEqual(address.Host, "10.0.0.2");
+                Assert.AreEqual(address.Port, 21);
+                Assert.IsNotNull(proxy.Credentials);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("http_proxy", cachedHttpProxy);
+            }
+        }
+
+        [TestMethod]
+        [TestCategory("UnitTest")]
+        [TestCategory("Runtime")]
+        public void TestProxyOverridingEnvironmentVariables()
+        {
+            var cachedHttpProxy = Environment.GetEnvironmentVariable("http_proxy");
+            var cachedHttpsProxy = Environment.GetEnvironmentVariable("https_proxy");
+
+            try
+            {
+                Environment.SetEnvironmentVariable("http_proxy", EnvironmentVariableUrl);
+                Environment.SetEnvironmentVariable("https_proxy", EnvironmentVariableUrl);
+
+                var dummyConfig = new AmazonEC2Config();
+                dummyConfig.ProxyHost = Host;
+                dummyConfig.ProxyPort = Port;
+
+                IWebProxy httpProxy = dummyConfig.GetHttpProxy();
+                var httpAddress = httpProxy.GetProxy(new Uri("https://ec2.us-west-2.aws.amazon.com"));
+                Assert.AreEqual(httpAddress.Host, Host);
+                Assert.AreEqual(httpAddress.Port, Port);
+
+                IWebProxy httpsProxy = dummyConfig.GetHttpsProxy();
+                var httpsAddress = httpsProxy.GetProxy(new Uri("https://ec2.us-west-2.aws.amazon.com"));
+                Assert.AreEqual(httpsAddress.Host, Host);
+                Assert.AreEqual(httpsAddress.Port, Port);
+            }
+            finally
+            {
+                Environment.SetEnvironmentVariable("http_proxy", cachedHttpProxy);
+                Environment.SetEnvironmentVariable("https_proxy", cachedHttpsProxy);
+            }
+        }
 #endif
     }
 }