Support Non proxy host settings in the ProxyConfiguration for Crt htt…

…p client. (#4962)

* Support Non proxy host settings in the ProxyConfiguration for Crt http client.

* Handled comments

.changes/next-release/feature-AWSCRTHTTPClient-69af591.json

@@ -0,0 +1,6 @@
+{
+    "type": "feature",
+    "category": "AWS CRT HTTP Client",
+    "contributor": "",
+    "description": "Support Non proxy host settings in the ProxyConfiguration for Crt http client."
+}

core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtConfigurationUtils.java

@@ -15,7 +15,11 @@
 
 package software.amazon.awssdk.crtcore;
 
+import static software.amazon.awssdk.utils.StringUtils.lowerCase;
+
+import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import software.amazon.awssdk.annotations.SdkProtectedApi;
 import software.amazon.awssdk.crt.http.HttpMonitoringOptions;
 import software.amazon.awssdk.crt.http.HttpProxyOptions;
@@ -33,7 +37,9 @@ public static Optional<HttpProxyOptions> resolveProxy(CrtProxyConfiguration prox
         if (proxyConfiguration == null) {
             return Optional.empty();
         }
-
+        if (doesTargetMatchNonProxyHosts(proxyConfiguration.host(), proxyConfiguration.nonProxyHosts())) {
+            return Optional.empty();
+        }
         HttpProxyOptions clientProxyOptions = new HttpProxyOptions();
 
         clientProxyOptions.setHost(proxyConfiguration.host());
@@ -54,11 +60,19 @@ public static Optional<HttpProxyOptions> resolveProxy(CrtProxyConfiguration prox
         return Optional.of(clientProxyOptions);
     }
 
+    private static boolean doesTargetMatchNonProxyHosts(String target, Set<String> hostPatterns) {
+        return Optional.ofNullable(hostPatterns)
+                       .map(patterns ->
+                                patterns.stream()
+                                        .filter(Objects::nonNull)
+                                        .anyMatch(pattern -> target != null && lowerCase(target).matches(pattern)))
+                       .orElse(false);
+    }
+
     public static Optional<HttpMonitoringOptions> resolveHttpMonitoringOptions(CrtConnectionHealthConfiguration config) {
         if (config == null) {
             return Optional.empty();
         }
-
         HttpMonitoringOptions httpMonitoringOptions = new HttpMonitoringOptions();
         httpMonitoringOptions.setMinThroughputBytesPerSecond(config.minimumThroughputInBps());
         int seconds = NumericUtils.saturatedCast(config.minimumThroughputTimeout().getSeconds());

core/crt-core/src/main/java/software/amazon/awssdk/crtcore/CrtProxyConfiguration.java

@@ -17,7 +17,10 @@
 
 import static software.amazon.awssdk.utils.ProxyConfigProvider.fromSystemEnvironmentSettings;
 
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 import software.amazon.awssdk.utils.ProxyConfigProvider;
 import software.amazon.awssdk.utils.ProxySystemSetting;
@@ -36,6 +39,7 @@ public abstract class CrtProxyConfiguration {
     private final String password;
     private final Boolean useSystemPropertyValues;
     private final Boolean useEnvironmentVariableValues;
+    private final Set<String> nonProxyHosts;
 
     protected CrtProxyConfiguration(DefaultBuilder<?> builder) {
         this.useSystemPropertyValues = builder.useSystemPropertyValues;
@@ -49,6 +53,7 @@ protected CrtProxyConfiguration(DefaultBuilder<?> builder) {
         this.port = resolvePort(builder, proxyConfigProvider);
         this.username = resolveUsername(builder, proxyConfigProvider);
         this.password = resolvePassword(builder, proxyConfigProvider);
+        this.nonProxyHosts = resolveNonProxyHosts(builder, proxyConfigProvider);
     }
 
     private static String resolvePassword(DefaultBuilder<?> builder, ProxyConfigProvider proxyConfigProvider) {
@@ -83,6 +88,13 @@ private static String resolveHost(DefaultBuilder<?> builder, ProxyConfigProvider
         }
     }
 
+    private Set<String> resolveNonProxyHosts(DefaultBuilder<?> builder, ProxyConfigProvider proxyConfigProvider) {
+        if (builder.nonProxyHosts != null || proxyConfigProvider == null) {
+            return builder.nonProxyHosts;
+        }
+        return proxyConfigProvider.nonProxyHosts();
+    }
+
     /**
      * @return The proxy scheme.
      */
@@ -132,6 +144,16 @@ public final Boolean isUseEnvironmentVariableValues() {
         return useEnvironmentVariableValues;
     }
 
+    /**
+     * Retrieves the hosts that the client is allowed to access without going through the proxy.
+     * If the value is not set on the object, the value represented by the environment variable or system property is returned.
+     *
+     * @see Builder#nonProxyHosts(Set)
+     */
+    public Set<String> nonProxyHosts() {
+        return Collections.unmodifiableSet(nonProxyHosts != null ? nonProxyHosts : Collections.emptySet());
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -162,7 +184,10 @@ public boolean equals(Object o) {
         if (!Objects.equals(useSystemPropertyValues, that.useSystemPropertyValues)) {
             return false;
         }
-        return Objects.equals(useEnvironmentVariableValues, that.useEnvironmentVariableValues);
+        if (!Objects.equals(useEnvironmentVariableValues, that.useEnvironmentVariableValues)) {
+            return false;
+        }
+        return Objects.equals(nonProxyHosts, that.nonProxyHosts);
     }
 
     @Override
@@ -175,6 +200,7 @@ public int hashCode() {
         result = 31 * result + (useSystemPropertyValues != null ? useSystemPropertyValues.hashCode() : 0);
         result = 31 * result + (useEnvironmentVariableValues != null ? useEnvironmentVariableValues.hashCode() : 0);
         result = 31 * result + (scheme != null ? scheme.hashCode() : 0);
+        result = 31 * result + (nonProxyHosts != null ? nonProxyHosts.hashCode() : 0);
         return result;
     }
 
@@ -253,6 +279,17 @@ public interface Builder {
          */
         Builder useEnvironmentVariableValues(Boolean useEnvironmentVariableValues);
 
+        /**
+         * Configure the hosts that the client is allowed to access without going through the proxy.
+         */
+        Builder nonProxyHosts(Set<String> nonProxyHosts);
+
+
+        /**
+         * Add a host that the client is allowed to access without going through the proxy.
+         */
+        Builder addNonProxyHost(String nonProxyHost);
+
 
         CrtProxyConfiguration build();
     }
@@ -266,6 +303,8 @@ protected abstract static class DefaultBuilder<B extends Builder> implements Bui
         private String password;
         private Boolean useSystemPropertyValues = Boolean.TRUE;
         private Boolean useEnvironmentVariableValues = Boolean.TRUE;
+        private Set<String> nonProxyHosts;
+
 
         protected DefaultBuilder() {
         }
@@ -278,6 +317,7 @@ protected DefaultBuilder(CrtProxyConfiguration proxyConfiguration) {
             this.port = proxyConfiguration.port;
             this.username = proxyConfiguration.username;
             this.password = proxyConfiguration.password;
+            this.nonProxyHosts = proxyConfiguration.nonProxyHosts;
         }
 
         @Override
@@ -322,6 +362,21 @@ public B useEnvironmentVariableValues(Boolean useEnvironmentVariableValues) {
             return (B) this;
         }
 
+        @Override
+        public B nonProxyHosts(Set<String> nonProxyHosts) {
+            this.nonProxyHosts = nonProxyHosts != null ? new HashSet<>(nonProxyHosts) : null;
+            return (B) this;
+        }
+
+        @Override
+        public B addNonProxyHost(String nonProxyHost) {
+            if (this.nonProxyHosts == null) {
+                this.nonProxyHosts = new HashSet<>();
+            }
+            this.nonProxyHosts.add(nonProxyHost);
+            return (B) this;
+        }
+
         public B setuseEnvironmentVariableValues(Boolean useEnvironmentVariableValues) {
             return useEnvironmentVariableValues(useEnvironmentVariableValues);
         }

core/crt-core/src/test/java/software/amazon/awssdk/crtcore/CrtConnectionUtilsTest.java

@@ -20,7 +20,11 @@
 
 import java.time.Duration;
 import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 import org.mockito.Mockito;
 import software.amazon.awssdk.crt.http.HttpMonitoringOptions;
 import software.amazon.awssdk.crt.http.HttpProxyOptions;
@@ -54,6 +58,60 @@ void resolveProxy_emptyProxy_shouldReturnEmpty() {
         assertThat(CrtConfigurationUtils.resolveProxy(null, tlsContext)).isEmpty();
     }
 
+    @ParameterizedTest
+    @ValueSource(strings = {".*?.2.3.4", "1.*?.3.4", ".*?"})
+    void resolveProxy_withSingleNonProxyHostsWidCards_shouldReturnEmpty(String nonProxyHost) {
+        TlsContext tlsContext = Mockito.mock(TlsContext.class);
+        CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4")
+                                                                     .port(123)
+                                                                     .scheme("https")
+                                                                     .password("bar")
+                                                                     .username("foo")
+                                                                     .nonProxyHosts(Stream.of(nonProxyHost,"someRandom")
+                                                                                          .collect(Collectors.toSet()))
+                                                                     .build();
+        assertThat(CrtConfigurationUtils.resolveProxy(configuration, tlsContext)).isEmpty();
+    }
+
+
+
+    @Test
+    void resolveProxy_withNullHostAndNonPorxy_shouldNotReturnEmpty( ) {
+        TlsContext tlsContext = Mockito.mock(TlsContext.class);
+        CrtProxyConfiguration configuration = new TestProxy.Builder().host(null)
+                                                                     .port(123)
+                                                                     .scheme("https")
+                                                                     .password("bar")
+                                                                     .username("foo")
+                                                                     .nonProxyHosts(Stream.of("someRandom", "null")
+                                                                                          .collect(Collectors.toSet()))
+                                                                     .build();
+        assertThat(CrtConfigurationUtils.resolveProxy(configuration, tlsContext)).isNotEmpty();
+    }
+
+    @Test
+    void resolveProxy_basicAuthorization_WithNonMatchingNoProxy() {
+        CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4")
+                                                                     .port(123)
+                                                                     .scheme("https")
+                                                                     .password("bar")
+                                                                     .addNonProxyHost("someRandom")
+                                                                     .addNonProxyHost(null)
+                                                                     .username("foo")
+                                                                     .build();
+
+        TlsContext tlsContext = Mockito.mock(TlsContext.class);
+
+        Optional<HttpProxyOptions> httpProxyOptions = CrtConfigurationUtils.resolveProxy(configuration, tlsContext);
+        assertThat(httpProxyOptions).hasValueSatisfying(proxy -> {
+            assertThat(proxy.getTlsContext()).isEqualTo(tlsContext);
+            assertThat(proxy.getAuthorizationPassword()).isEqualTo("bar");
+            assertThat(proxy.getAuthorizationUsername()).isEqualTo("foo");
+            assertThat(proxy.getAuthorizationType()).isEqualTo(HttpProxyOptions.HttpProxyAuthorizationType.Basic);
+        });
+    }
+
+
     @Test
     void resolveProxy_noneAuthorization() {
         CrtProxyConfiguration configuration = new TestProxy.Builder().host("1.2.3.4")

http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/ProxyConfiguration.java

@@ -15,8 +15,10 @@
 
 package software.amazon.awssdk.http.crt;
 
+import java.util.Set;
 import software.amazon.awssdk.annotations.SdkPublicApi;
 import software.amazon.awssdk.crtcore.CrtProxyConfiguration;
+import software.amazon.awssdk.utils.ProxyEnvironmentSetting;
 import software.amazon.awssdk.utils.ProxySystemSetting;
 import software.amazon.awssdk.utils.builder.CopyableBuilder;
 import software.amazon.awssdk.utils.builder.ToCopyableBuilder;
@@ -116,9 +118,35 @@ public interface Builder extends CrtProxyConfiguration.Builder, CopyableBuilder<
         Builder useSystemPropertyValues(Boolean useSystemPropertyValues);
 
 
+        /**
+         * Set the option whether to use environment variable values for {@link ProxyEnvironmentSetting} if any of the config
+         * options are missing. The value is set to "true" by default, enabling the SDK to automatically use environment variable
+         * values for proxy configuration options that are not provided during building the {@link ProxyConfiguration} object. To
+         * disable this behavior, set this value to "false".It is important to note that when this property is set to "true," all
+         * proxy settings will exclusively originate from Environment Variable Values, and no partial settings will be obtained
+         * from System Property Values.
+         * <p>Comma-separated host names in the NO_PROXY environment variable indicate multiple hosts to exclude from
+         * proxy settings.
+         *
+         * @param useEnvironmentVariableValues The option whether to use environment variable values
+         * @return This object for method chaining.
+         */
         @Override
         Builder useEnvironmentVariableValues(Boolean useEnvironmentVariableValues);
 
+        /**
+         * Configure the hosts that the client is allowed to access without going through the proxy.
+         */
+        @Override
+        Builder nonProxyHosts(Set<String> nonProxyHosts);
+
+
+        /**
+         * Add a host that the client is allowed to access without going through the proxy.
+         */
+        @Override
+        Builder addNonProxyHost(String nonProxyHost);
+
         @Override
         ProxyConfiguration build();
     }

http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/CrtHttpProxyTest.java

@@ -18,6 +18,7 @@
 import static org.assertj.core.api.Assertions.assertThat;
 
 import java.net.URISyntaxException;
+import java.util.Set;
 import software.amazon.awssdk.http.HttpProxyTestSuite;
 import software.amazon.awssdk.http.proxy.TestProxySetting;
 
@@ -35,6 +36,7 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings,
             Integer portNumber = userSetProxySettings.getPort();
             String userName = userSetProxySettings.getUserName();
             String password = userSetProxySettings.getPassword();
+            Set<String> nonProxyHosts = userSetProxySettings.getNonProxyHosts();
 
             if (hostName != null) {
                 proxyBuilder.host(hostName);
@@ -48,6 +50,9 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings,
             if (password != null) {
                 proxyBuilder.password(password);
             }
+            if (nonProxyHosts != null && !nonProxyHosts.isEmpty()) {
+                proxyBuilder.nonProxyHosts(nonProxyHosts);
+            }
         }
 
         if (!"http".equals(protocol)) {
@@ -64,6 +69,7 @@ protected void assertProxyConfiguration(TestProxySetting userSetProxySettings,
         assertThat(proxyConfiguration.port()).isEqualTo(expectedProxySettings.getPort());
         assertThat(proxyConfiguration.username()).isEqualTo(expectedProxySettings.getUserName());
         assertThat(proxyConfiguration.password()).isEqualTo(expectedProxySettings.getPassword());
+        assertThat(proxyConfiguration.nonProxyHosts()).isEqualTo(expectedProxySettings.getNonProxyHosts());
     }
 
 }

http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/ProxyConfigurationTest.java

@@ -19,7 +19,11 @@
 
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.util.HashSet;
 import java.util.Random;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeEach;
@@ -181,6 +185,8 @@ private void setRandomValue(Object o, Method setter) throws InvocationTargetExce
             setter.invoke(o, RNG.nextInt());
         } else if (Boolean.class.equals(paramClass)) {
             setter.invoke(o, RNG.nextBoolean());
+        } else if (Set.class.equals(paramClass)) {
+            setter.invoke(o, IntStream.range(0, 5).mapToObj(i -> randomString()).collect(Collectors.toSet()));
         } else {
             throw new RuntimeException("Don't know how create random value for type " + paramClass);
         }