From a0b30bf6a50c9656764a45a30d98b489dc189222 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sun, 5 Nov 2023 20:42:22 +0100 Subject: [PATCH 01/10] #2296 --- karate-core/pom.xml | 11 +- .../intuit/karate/http/ApacheHttpClient.java | 277 ++++++++----- .../intuit/karate/http/ApacheHttpClient4.java | 372 ++++++++++++++++++ .../http/CustomHttpRequestRetryHandler.java | 31 +- .../LenientSslConnectionSocketFactory.java | 49 +++ .../karate/fatjar/ProxyServerSslTest.java | 38 +- .../intuit/karate/fatjar/ProxyServerTest.java | 33 +- 7 files changed, 680 insertions(+), 131 deletions(-) create mode 100644 karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java create mode 100644 karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java diff --git a/karate-core/pom.xml b/karate-core/pom.xml index 51a912a9a..3a725cea1 100644 --- a/karate-core/pom.xml +++ b/karate-core/pom.xml @@ -36,7 +36,8 @@ com.linecorp.armeria armeria 1.25.2 - + + org.apache.httpcomponents httpclient @@ -53,7 +54,13 @@ commons-codec commons-codec 1.16.0 - + + + + org.apache.httpcomponents.client5 + httpclient5 + 5.2.1 + ch.qos.logback logback-classic diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 3b2833964..1f7a080ef 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -28,6 +28,7 @@ import com.intuit.karate.Logger; import com.intuit.karate.core.Config; import com.intuit.karate.core.ScenarioEngine; + import io.netty.handler.codec.http.cookie.ClientCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; import java.io.IOException; @@ -47,47 +48,54 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.TimeUnit; + import javax.net.ssl.SSLContext; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpHost; -import org.apache.http.HttpMessage; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.NTCredentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.CookieStore; -import org.apache.http.client.CredentialsProvider; + +import org.apache.hc.client5.http.ClientProtocolException; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.NTCredentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.cookie.BasicCookieStore; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookieSpecFactory; +import org.apache.hc.client5.http.cookie.CookieStore; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.entity.EntityBuilder; +import org.apache.hc.client5.http.impl.DefaultRedirectStrategy; +import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.client5.http.impl.cookie.CookieSpecBase; +import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; +import org.apache.hc.client5.http.routing.HttpRoutePlanner; +import org.apache.hc.client5.http.ssl.LenientSslConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.TrustAllStrategy; +import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy; +import org.apache.hc.core5.http.EntityDetails; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpMessage; +import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.config.Registry; +import org.apache.hc.core5.http.config.RegistryBuilder; +import org.apache.hc.core5.http.io.SocketConfig; +import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.ssl.SSLContextBuilder; +import org.apache.hc.core5.ssl.SSLContexts; import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.EntityBuilder; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.ssl.LenientSslConnectionSocketFactory; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.CookieSpecProvider; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.LaxRedirectStrategy; -import org.apache.http.impl.conn.SystemDefaultRoutePlanner; -import org.apache.http.impl.cookie.DefaultCookieSpec; -import org.apache.http.protocol.HttpContext; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; /** * @@ -95,6 +103,8 @@ */ public class ApacheHttpClient implements HttpClient, HttpRequestInterceptor { + private static final AuthScope ANY_AUTH_SCOPE = new AuthScope(null, null, -1, null, null); + private final ScenarioEngine engine; private final Logger logger; private final HttpLogger httpLogger; @@ -102,12 +112,19 @@ public class ApacheHttpClient implements HttpClient, HttpRequestInterceptor { private HttpClientBuilder clientBuilder; private CookieStore cookieStore; - public static class LenientCookieSpec extends DefaultCookieSpec { + // Not sure what the rationale was behind this class. + // But the httpclient4 ApacheHttpClient, based on DefaultCookieSpec, supported: + // - set-cookie2 which is now deprecated https://stackoverflow.com/questions/9462180/difference-between-set-cookie2-and-set-cookie + // - "netscape style cookies" and versioned cookies... whatever that was, I'm asusming its not widely used any more + // - other than that, it defaulted to a RFC2965Strict Spec. + // So as part of the httpclient5 migration, we directly default to RFC6265StrictSpec + public static class LenientCookieSpec extends CookieSpecBase { static final String KARATE = "karate"; + final RFC6265StrictSpec strict = new RFC6265StrictSpec(); + public LenientCookieSpec() { - super(new String[]{"EEE, dd-MMM-yy HH:mm:ss z", "EEE, dd MMM yyyy HH:mm:ss Z"}, false); } @Override @@ -120,12 +137,22 @@ public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieE // do nothing } - public static Registry registry() { - CookieSpecProvider specProvider = (HttpContext hc) -> new LenientCookieSpec(); - return RegistryBuilder.create() - .register(KARATE, specProvider).build(); + + @Override + public List parse(Header header, CookieOrigin origin) throws MalformedCookieException { + return strict.parse(header, origin); + } + + @Override + public List
formatCookies(List cookies) { + return strict.formatCookies(cookies); } + public static Registry registry() { + CookieSpecFactory specProvider = (HttpContext hc) -> new LenientCookieSpec(); + return RegistryBuilder.create() + .register(KARATE, specProvider).build(); + } } public ApacheHttpClient(ScenarioEngine engine) { @@ -136,17 +163,22 @@ public ApacheHttpClient(ScenarioEngine engine) { } private void configure(Config config) { + PoolingHttpClientConnectionManagerBuilder connectionManagerBuilder = PoolingHttpClientConnectionManagerBuilder.create(); + clientBuilder = HttpClientBuilder.create(); + if (config.isHttpRetryEnabled()) { - clientBuilder.setRetryHandler(new CustomHttpRequestRetryHandler(logger)); + clientBuilder.setRetryStrategy(new CustomHttpRequestRetryHandler(logger)); } else { clientBuilder.disableAutomaticRetries(); } if (!config.isFollowRedirects()) { clientBuilder.disableRedirectHandling(); - } else { // support redirect on POST by default - clientBuilder.setRedirectStrategy(LaxRedirectStrategy.INSTANCE); + } else { + clientBuilder.setRedirectStrategy(DefaultRedirectStrategy.INSTANCE); + // httpclient4 was using LaxRedirectStrategy.INSTANCE as it supported redirect on POST methods. + // httpclient5 seems to be status code based, not method based, so default strategy should be fine. } cookieStore = new BasicCookieStore(); clientBuilder.setDefaultCookieStore(cookieStore); @@ -181,71 +213,89 @@ private void configure(Config config) { } else { socketFactory = new LenientSslConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); } - clientBuilder.setSSLSocketFactory(socketFactory); + connectionManagerBuilder.setSSLSocketFactory(socketFactory); } catch (Exception e) { logger.error("ssl context init failed: {}", e.getMessage()); throw new RuntimeException(e); } } + connectionManagerBuilder.setDefaultConnectionConfig(ConnectionConfig.custom() + .setSocketTimeout(config.getReadTimeout(), TimeUnit.MILLISECONDS) + .setConnectTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); RequestConfig.Builder configBuilder = RequestConfig.custom() - .setCookieSpec(LenientCookieSpec.KARATE) - .setConnectTimeout(config.getConnectTimeout()) - .setSocketTimeout(config.getReadTimeout()); - if (config.getLocalAddress() != null) { - try { - InetAddress localAddress = InetAddress.getByName(config.getLocalAddress()); - configBuilder.setLocalAddress(localAddress); - } catch (Exception e) { - logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); - } - } + .setCookieSpec(LenientCookieSpec.KARATE); if (config.isNtlmEnabled()) { List authSchemes = new ArrayList<>(); authSchemes.add(AuthSchemes.NTLM); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); NTCredentials ntCredentials = new NTCredentials( - config.getNtlmUsername(), config.getNtlmPassword(), config.getNtlmWorkstation(), config.getNtlmDomain()); - credentialsProvider.setCredentials(AuthScope.ANY, ntCredentials); + config.getNtlmUsername(), config.getNtlmPassword().toCharArray(), config.getNtlmWorkstation(), config.getNtlmDomain()); + credentialsProvider.setCredentials(ANY_AUTH_SCOPE, ntCredentials); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); configBuilder.setTargetPreferredAuthSchemes(authSchemes); } clientBuilder.setDefaultRequestConfig(configBuilder.build()); - SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout()); - clientBuilder.setDefaultSocketConfig(socketBuilder.build()); + SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS); + connectionManagerBuilder.setDefaultSocketConfig(socketBuilder.build()); + + clientBuilder.setRoutePlanner(buildRoutePlanner(config)); + clientBuilder.setConnectionManager(connectionManagerBuilder.build()); + + clientBuilder.addRequestInterceptorLast(this); + } + + // Differences with httpclient4 implementation: + // - RequestBuilder.setLocalAddress does not exist any more, so instead, RoutePlanner.determineLocalAddress is overridden + // - clientBuilder.setProxy is not set any more. I'm probably misreading the code, but looking at DefaultRoutePanner.determineRoute, if proxy exists, my understanding is that determineProxy is not called and therefore proxySelector will NOT be used. + // so ProxySelector has been redesigned to handle both the specified proxy, and the nonProxyhosts if specified. + // Note that the route planner must now handle: + // - localaddress + // - proxy (set or not) + // - nonProxy hosts (set or not) + // Only SystemDefaultRoutePlanner supports proxy/nonProxy host so we subclass that class. However, SystemDefaultRoutePlanner does not have a "no proxy" mode so determineProxy is overridden to opt out if needed. + private HttpRoutePlanner buildRoutePlanner(Config config) { + ProxySelector proxySelector = null; if (config.getProxyUri() != null) { try { URI proxyUri = new URIBuilder(config.getProxyUri()).build(); - clientBuilder.setProxy(new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme())); + + proxySelector = new ProxySelector() { + private final List proxyExceptions = config.getNonProxyHosts() == null ? Collections.emptyList() : config.getNonProxyHosts(); + + @Override + public List select(URI uri) { + return Collections.singletonList(proxyExceptions.contains(uri.getHost()) + ? Proxy.NO_PROXY + : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); + } + + @Override + public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { + logger.info("connect failed to uri: {}", uri, ioe); + } + }; + if (config.getProxyUsername() != null && config.getProxyPassword() != null) { - CredentialsProvider credsProvider = new BasicCredentialsProvider(); + BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); credsProvider.setCredentials( new AuthScope(proxyUri.getHost(), proxyUri.getPort()), - new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); + new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword().toCharArray())); clientBuilder.setDefaultCredentialsProvider(credsProvider); } - if (config.getNonProxyHosts() != null) { - ProxySelector proxySelector = new ProxySelector() { - private final List proxyExceptions = config.getNonProxyHosts(); - - @Override - public List select(URI uri) { - return Collections.singletonList(proxyExceptions.contains(uri.getHost()) - ? Proxy.NO_PROXY - : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); - } - - @Override - public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { - logger.info("connect failed to uri: {}", uri, ioe); - } - }; - clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(proxySelector)); - } } catch (Exception e) { throw new RuntimeException(e); } } - clientBuilder.addInterceptorLast(this); + InetAddress localAddress = null; + if (config.getLocalAddress() != null) { + try { + localAddress = InetAddress.getByName(config.getLocalAddress()); + } catch (Exception e) { + logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); + } + } + + return new CustomRoutePlanner(proxySelector, localAddress); } @Override @@ -263,7 +313,7 @@ public Config getConfig() { @Override public Response invoke(HttpRequest request) { this.request = request; - RequestBuilder requestBuilder = RequestBuilder.create(request.getMethod()).setUri(request.getUrl()); + ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.create(request.getMethod()).setUri(request.getUrl()); if (request.getBody() != null) { EntityBuilder entityBuilder = EntityBuilder.create().setBinary(request.getBody()); List transferEncoding = request.getHeaderValues(HttpConstants.HDR_TRANSFER_ENCODING); @@ -276,7 +326,7 @@ public Response invoke(HttpRequest request) { entityBuilder.chunked(); } if (te.contains("gzip")) { - entityBuilder.gzipCompress(); + entityBuilder.gzipCompressed(); } } request.removeHeader(HttpConstants.HDR_TRANSFER_ENCODING); @@ -306,14 +356,14 @@ public Response invoke(HttpRequest request) { throw new RuntimeException(e); } } - int statusCode = httpResponse.getStatusLine().getStatusCode(); + int statusCode = httpResponse.getCode(); Map> headers = toHeaders(httpResponse); List storedCookies = cookieStore.getCookies(); Header[] requestCookieHeaders = httpResponse.getHeaders(HttpConstants.HDR_SET_COOKIE); // edge case where the apache client // auto-followed a redirect where cookies were involved - List mergedCookieValues = new ArrayList(requestCookieHeaders.length); - Set alreadyMerged = new HashSet(requestCookieHeaders.length); + List mergedCookieValues = new ArrayList<>(requestCookieHeaders.length); + Set alreadyMerged = new HashSet<>(requestCookieHeaders.length); for (Header ch : requestCookieHeaders) { String requestCookieValue = ch.getValue(); io.netty.handler.codec.http.cookie.Cookie c = ClientCookieDecoder.LAX.decode(requestCookieValue); @@ -326,7 +376,7 @@ public Response invoke(HttpRequest request) { if (alreadyMerged.contains(name)) { continue; } - Map map = new HashMap(); + Map map = new HashMap<>(); map.put(Cookies.NAME, name); map.put(Cookies.VALUE, c.getValue()); map.put(Cookies.DOMAIN, c.getDomain()); @@ -347,19 +397,19 @@ public Response invoke(HttpRequest request) { } @Override - public void process(org.apache.http.HttpRequest hr, HttpContext hc) throws HttpException, IOException { + public void process(org.apache.hc.core5.http.HttpRequest hr, EntityDetails entity, HttpContext context) throws HttpException, IOException { request.setHeaders(toHeaders(hr)); httpLogger.logRequest(getConfig(), request); request.setStartTime(System.currentTimeMillis()); } private static Map> toHeaders(HttpMessage msg) { - Header[] headers = msg.getAllHeaders(); - Map> map = new LinkedHashMap(headers.length); + Header[] headers = msg.getHeaders(); + Map> map = new LinkedHashMap<>(headers.length); for (Header outer : headers) { String name = outer.getName(); Header[] inner = msg.getHeaders(name); - List list = new ArrayList(inner.length); + List list = new ArrayList<>(inner.length); for (Header h : inner) { list.add(h.getValue()); } @@ -368,4 +418,37 @@ private static Map> toHeaders(HttpMessage msg) { return map; } + private static class CustomRoutePlanner extends SystemDefaultRoutePlanner { + + private final InetAddress localAddress; + private ProxySelector proxySelector; + + public CustomRoutePlanner(ProxySelector proxySelector, InetAddress localAddress) { + super(proxySelector); + // ProxySelector in SystemDefaultRoutePlanner is private; + this.proxySelector = proxySelector; + this.localAddress = localAddress; + } + + @Override + protected HttpHost determineProxy( + final HttpHost target, + final HttpContext context) throws HttpException { + if (proxySelector == null) { + //SystemDefaultRoutePlanner will default to some system-wide proxySelector. + //However, the expected behavior here is to ignore proxy altogether if no selector is supplied. + return null; + } + return super.determineProxy(target, context); + } + + protected InetAddress determineLocalAddress( + final HttpHost firstHop, + final HttpContext context) throws HttpException { + return localAddress; + } + + + + } } diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java new file mode 100644 index 000000000..8feb9a572 --- /dev/null +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java @@ -0,0 +1,372 @@ +/* + * The MIT License + * + * Copyright 2022 Karate Labs Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package com.intuit.karate.http; + +import com.intuit.karate.Constants; +import com.intuit.karate.FileUtils; +import com.intuit.karate.Logger; +import com.intuit.karate.core.Config; +import com.intuit.karate.core.ScenarioEngine; +import io.netty.handler.codec.http.cookie.ClientCookieDecoder; +import io.netty.handler.codec.http.cookie.ServerCookieEncoder; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.Proxy; +import java.net.ProxySelector; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.URI; +import java.security.KeyStore; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import javax.net.ssl.SSLContext; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpHost; +import org.apache.http.HttpMessage; +import org.apache.http.HttpRequestInterceptor; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.NTCredentials; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CookieStore; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.config.AuthSchemes; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.entity.EntityBuilder; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.config.Registry; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.config.SocketConfig; +import org.apache.http.conn.ssl.LenientSslConnectionSocketFactory; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustAllStrategy; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.cookie.Cookie; +import org.apache.http.cookie.CookieOrigin; +import org.apache.http.cookie.CookieSpecProvider; +import org.apache.http.cookie.MalformedCookieException; +import org.apache.http.impl.client.BasicCookieStore; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.LaxRedirectStrategy; +import org.apache.http.impl.conn.SystemDefaultRoutePlanner; +import org.apache.http.impl.cookie.DefaultCookieSpec; +import org.apache.http.protocol.HttpContext; +import org.apache.http.ssl.SSLContextBuilder; +import org.apache.http.ssl.SSLContexts; + +/** + * + * @author pthomas3 + */ +public class ApacheHttpClient4 implements HttpClient, HttpRequestInterceptor { + + private final ScenarioEngine engine; + private final Logger logger; + private final HttpLogger httpLogger; + + private HttpClientBuilder clientBuilder; + private CookieStore cookieStore; + + public static class LenientCookieSpec extends DefaultCookieSpec { + + static final String KARATE = "karate"; + + public LenientCookieSpec() { + super(new String[]{"EEE, dd-MMM-yy HH:mm:ss z", "EEE, dd MMM yyyy HH:mm:ss Z"}, false); + } + + @Override + public boolean match(Cookie cookie, CookieOrigin origin) { + return true; + } + + @Override + public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException { + // do nothing + } + + public static Registry registry() { + CookieSpecProvider specProvider = (HttpContext hc) -> new LenientCookieSpec(); + return RegistryBuilder.create() + .register(KARATE, specProvider).build(); + } + + } + + public ApacheHttpClient4(ScenarioEngine engine) { + this.engine = engine; + logger = engine.logger; + httpLogger = new HttpLogger(logger); + configure(engine.getConfig()); + } + + private void configure(Config config) { + clientBuilder = HttpClientBuilder.create(); + if (config.isHttpRetryEnabled()) { + clientBuilder.setRetryHandler(new CustomHttpRequestRetryHandler(logger)); + } else { + clientBuilder.disableAutomaticRetries(); + } + + if (!config.isFollowRedirects()) { + clientBuilder.disableRedirectHandling(); + } else { // support redirect on POST by default + clientBuilder.setRedirectStrategy(LaxRedirectStrategy.INSTANCE); + } + cookieStore = new BasicCookieStore(); + clientBuilder.setDefaultCookieStore(cookieStore); + clientBuilder.setDefaultCookieSpecRegistry(LenientCookieSpec.registry()); + clientBuilder.useSystemProperties(); + if (config.isSslEnabled()) { + // System.setProperty("jsse.enableSNIExtension", "false"); + String algorithm = config.getSslAlgorithm(); // could be null + KeyStore trustStore = engine.getKeyStore(config.getSslTrustStore(), config.getSslTrustStorePassword(), config.getSslTrustStoreType()); + KeyStore keyStore = engine.getKeyStore(config.getSslKeyStore(), config.getSslKeyStorePassword(), config.getSslKeyStoreType()); + SSLContext sslContext; + try { + SSLContextBuilder builder = SSLContexts.custom() + .setProtocol(algorithm); // will default to TLS if null + if (trustStore == null && config.isSslTrustAll()) { + builder = builder.loadTrustMaterial(new TrustAllStrategy()); + } else { + if (config.isSslTrustAll()) { + builder = builder.loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()); + } else { + builder = builder.loadTrustMaterial(trustStore, null); // will use system / java default + } + } + if (keyStore != null) { + char[] keyPassword = config.getSslKeyStorePassword() == null ? null : config.getSslKeyStorePassword().toCharArray(); + builder = builder.loadKeyMaterial(keyStore, keyPassword); + } + sslContext = builder.build(); + SSLConnectionSocketFactory socketFactory; + if (keyStore != null) { + socketFactory = new SSLConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); + } else { + socketFactory = new LenientSslConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); + } + clientBuilder.setSSLSocketFactory(socketFactory); + } catch (Exception e) { + logger.error("ssl context init failed: {}", e.getMessage()); + throw new RuntimeException(e); + } + } + RequestConfig.Builder configBuilder = RequestConfig.custom() + .setCookieSpec(LenientCookieSpec.KARATE) + .setConnectTimeout(config.getConnectTimeout()) + .setSocketTimeout(config.getReadTimeout()); + if (config.getLocalAddress() != null) { + try { + InetAddress localAddress = InetAddress.getByName(config.getLocalAddress()); + configBuilder.setLocalAddress(localAddress); + } catch (Exception e) { + logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); + } + } + if (config.isNtlmEnabled()) { + List authSchemes = new ArrayList<>(); + authSchemes.add(AuthSchemes.NTLM); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + NTCredentials ntCredentials = new NTCredentials( + config.getNtlmUsername(), config.getNtlmPassword(), config.getNtlmWorkstation(), config.getNtlmDomain()); + credentialsProvider.setCredentials(AuthScope.ANY, ntCredentials); + clientBuilder.setDefaultCredentialsProvider(credentialsProvider); + configBuilder.setTargetPreferredAuthSchemes(authSchemes); + } + clientBuilder.setDefaultRequestConfig(configBuilder.build()); + SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout()); + clientBuilder.setDefaultSocketConfig(socketBuilder.build()); + if (config.getProxyUri() != null) { + try { + URI proxyUri = new URIBuilder(config.getProxyUri()).build(); + clientBuilder.setProxy(new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme())); + if (config.getProxyUsername() != null && config.getProxyPassword() != null) { + CredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(proxyUri.getHost(), proxyUri.getPort()), + new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); + clientBuilder.setDefaultCredentialsProvider(credsProvider); + } + if (config.getNonProxyHosts() != null) { + ProxySelector proxySelector = new ProxySelector() { + private final List proxyExceptions = config.getNonProxyHosts(); + + @Override + public List select(URI uri) { + return Collections.singletonList(proxyExceptions.contains(uri.getHost()) + ? Proxy.NO_PROXY + : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); + } + + @Override + public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { + logger.info("connect failed to uri: {}", uri, ioe); + } + }; + clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(proxySelector)); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + clientBuilder.addInterceptorLast(this); + } + + @Override + public void setConfig(Config config) { + configure(config); + } + + @Override + public Config getConfig() { + return engine.getConfig(); + } + + private HttpRequest request; + + @Override + public Response invoke(HttpRequest request) { + this.request = request; + RequestBuilder requestBuilder = RequestBuilder.create(request.getMethod()).setUri(request.getUrl()); + if (request.getBody() != null) { + EntityBuilder entityBuilder = EntityBuilder.create().setBinary(request.getBody()); + List transferEncoding = request.getHeaderValues(HttpConstants.HDR_TRANSFER_ENCODING); + if (transferEncoding != null) { + for (String te : transferEncoding) { + if (te == null) { + continue; + } + if (te.contains("chunked")) { // can be comma delimited as per spec + entityBuilder.chunked(); + } + if (te.contains("gzip")) { + entityBuilder.gzipCompress(); + } + } + request.removeHeader(HttpConstants.HDR_TRANSFER_ENCODING); + } + requestBuilder.setEntity(entityBuilder.build()); + } + if (request.getHeaders() != null) { + request.getHeaders().forEach((k, vals) -> vals.forEach(v -> requestBuilder.addHeader(k, v))); + } + CloseableHttpResponse httpResponse; + byte[] bytes; + try (CloseableHttpClient client = clientBuilder.build()) { + httpResponse = client.execute(requestBuilder.build()); + HttpEntity responseEntity = httpResponse.getEntity(); + if (responseEntity == null || responseEntity.getContent() == null) { + bytes = Constants.ZERO_BYTES; + } else { + InputStream is = responseEntity.getContent(); + bytes = FileUtils.toBytes(is); + } + request.setEndTime(System.currentTimeMillis()); + httpResponse.close(); + } catch (Exception e) { + if (e instanceof ClientProtocolException && e.getCause() != null) { // better error message + throw new RuntimeException(e.getCause()); + } else { + throw new RuntimeException(e); + } + } + int statusCode = httpResponse.getStatusLine().getStatusCode(); + Map> headers = toHeaders(httpResponse); + List storedCookies = cookieStore.getCookies(); + Header[] requestCookieHeaders = httpResponse.getHeaders(HttpConstants.HDR_SET_COOKIE); + // edge case where the apache client + // auto-followed a redirect where cookies were involved + List mergedCookieValues = new ArrayList(requestCookieHeaders.length); + Set alreadyMerged = new HashSet(requestCookieHeaders.length); + for (Header ch : requestCookieHeaders) { + String requestCookieValue = ch.getValue(); + io.netty.handler.codec.http.cookie.Cookie c = ClientCookieDecoder.LAX.decode(requestCookieValue); + mergedCookieValues.add(requestCookieValue); + alreadyMerged.add(c.name()); + } + for (Cookie c : storedCookies) { + if (c.getValue() != null) { + String name = c.getName(); + if (alreadyMerged.contains(name)) { + continue; + } + Map map = new HashMap(); + map.put(Cookies.NAME, name); + map.put(Cookies.VALUE, c.getValue()); + map.put(Cookies.DOMAIN, c.getDomain()); + if (c.getExpiryDate() != null) { + map.put(Cookies.MAX_AGE, c.getExpiryDate().getTime()); + } + map.put(Cookies.SECURE, c.isSecure()); + io.netty.handler.codec.http.cookie.Cookie nettyCookie = Cookies.fromMap(map); + String cookieValue = ServerCookieEncoder.LAX.encode(nettyCookie); + mergedCookieValues.add(cookieValue); + } + } + headers.put(HttpConstants.HDR_SET_COOKIE, mergedCookieValues); + cookieStore.clear(); + Response response = new Response(statusCode, headers, bytes); + httpLogger.logResponse(getConfig(), request, response); + return response; + } + + @Override + public void process(org.apache.http.HttpRequest hr, HttpContext hc) throws HttpException, IOException { + request.setHeaders(toHeaders(hr)); + httpLogger.logRequest(getConfig(), request); + request.setStartTime(System.currentTimeMillis()); + } + + private static Map> toHeaders(HttpMessage msg) { + Header[] headers = msg.getAllHeaders(); + Map> map = new LinkedHashMap(headers.length); + for (Header outer : headers) { + String name = outer.getName(); + Header[] inner = msg.getHeaders(name); + List list = new ArrayList(inner.length); + for (Header h : inner) { + list.add(h.getValue()); + } + map.put(name, list); + } + return map; + } + +} diff --git a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java index 2157cf10c..6a61af6bf 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java +++ b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java @@ -2,6 +2,10 @@ import java.io.IOException; +import org.apache.hc.client5.http.HttpRequestRetryStrategy; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.util.TimeValue; import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.protocol.HttpContext; @@ -13,7 +17,7 @@ * This is usually the case when there is steal connection. The retry cause that * the connection is renewed and the second call will succeed. */ -public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler +public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler, HttpRequestRetryStrategy { private final Logger logger; @@ -23,8 +27,11 @@ public CustomHttpRequestRetryHandler(Logger logger) } @Override - public boolean retryRequest(IOException exception, int executionCount, HttpContext context) - { + public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { + return shouldRetry(exception, executionCount); + } + + private boolean shouldRetry(IOException exception, int executionCount) { if (exception instanceof NoHttpResponseException && executionCount < 1) { logger.error("Thrown an NoHttpResponseException retry..."); @@ -36,4 +43,22 @@ public boolean retryRequest(IOException exception, int executionCount, HttpConte return false; } } + + @Override + public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, + org.apache.hc.core5.http.protocol.HttpContext context) { + return shouldRetry(exception, executionCount); + } + + @Override + public boolean retryRequest(HttpResponse response, int execCount, + org.apache.hc.core5.http.protocol.HttpContext context) { + return false; + } + + @Override + public TimeValue getRetryInterval(HttpResponse response, int execCount, + org.apache.hc.core5.http.protocol.HttpContext context) { + return TimeValue.ofSeconds(1); // NOt sure what the interval was in httpclient4 ... Sticking with the default value of the default http5 implementation. + } } \ No newline at end of file diff --git a/karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java b/karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java new file mode 100644 index 000000000..f42a80839 --- /dev/null +++ b/karate-core/src/main/java/org/apache/hc/client5/http/ssl/LenientSslConnectionSocketFactory.java @@ -0,0 +1,49 @@ +/* + * The MIT License + * + * Copyright 2022 Karate Labs Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +package org.apache.hc.client5.http.ssl; + +import java.io.IOException; +import java.net.Socket; +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLContext; + +import org.apache.hc.core5.http.protocol.HttpContext; + +/** + * in a separate package just for log level config consistency + * + * @author pthomas3 + */ +public class LenientSslConnectionSocketFactory extends SSLConnectionSocketFactory { + + public LenientSslConnectionSocketFactory(SSLContext sslContext, HostnameVerifier hostnameVerifier) { + super(sslContext, hostnameVerifier); + } + + @Override + public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context) throws IOException { + return super.createLayeredSocket(socket, "", port, context); + } + +} diff --git a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java index 1ab3c9169..5eb56ffab 100644 --- a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java +++ b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerSslTest.java @@ -10,17 +10,20 @@ import java.nio.charset.StandardCharsets; import javax.net.ssl.SSLContext; import javax.net.ssl.TrustManager; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactoryBuilder; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -81,15 +84,18 @@ int http(HttpUriRequest request) throws Exception { SSLContext sc = SSLContext.getInstance("SSL"); sc.init(null, new TrustManager[]{LenientTrustManager.INSTANCE}, null); CloseableHttpClient client = HttpClients.custom() - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) - .setSSLContext(sc) + .setConnectionManager(PoolingHttpClientConnectionManagerBuilder.create() + .setSSLSocketFactory(SSLConnectionSocketFactoryBuilder.create() + .setSslContext(sc) + .setHostnameVerifier(NoopHostnameVerifier.INSTANCE) + .build()) + .build()) .setProxy(new HttpHost("localhost", proxy.getPort())) .build(); HttpResponse response = client.execute(request); - InputStream is = response.getEntity().getContent(); - String responseString = FileUtils.toString(is); + String responseString = response.getReasonPhrase(); logger.debug("response: {}", responseString); - return response.getStatusLine().getStatusCode(); + return response.getCode(); } } diff --git a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java index 171ba094a..06a1eee2c 100644 --- a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java +++ b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java @@ -7,16 +7,17 @@ import com.intuit.karate.core.MockServer; import java.io.InputStream; import java.nio.charset.StandardCharsets; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; + +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.classic.methods.HttpUriRequest; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.io.entity.StringEntity; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -37,11 +38,14 @@ class ProxyServerTest { @BeforeAll static void beforeAll() { proxy = new ProxyServer(0, null, null); + System.out.println("-3"); server = MockServer .feature("classpath:com/intuit/karate/fatjar/server.feature") .pathPrefix("/v1") .http(0).build(); + System.out.println("-2"); int port = server.getPort(); + System.out.println("-1"); System.setProperty("karate.server.port", port + ""); System.setProperty("karate.server.ssl", ""); // for ci System.setProperty("karate.server.proxy", "http://localhost:" + proxy.getPort()); @@ -55,9 +59,13 @@ static void afterAll() { @Test void testProxy() throws Exception { + System.out.println("0"); String url = "http://localhost:" + server.getPort() + "/v1/cats"; + System.out.println("1"); assertEquals(200, http(get(url))); + System.out.println("2"); assertEquals(200, http(post(url, "{ \"name\": \"Billie\" }"))); + System.out.println("3"); Results results = Runner .path("classpath:com/intuit/karate/fatjar/client.feature") .configDir("classpath:com/intuit/karate/fatjar") @@ -81,10 +89,9 @@ static int http(HttpUriRequest request) throws Exception { .setProxy(new HttpHost("localhost", proxy.getPort())) .build(); HttpResponse response = client.execute(request); - InputStream is = response.getEntity().getContent(); - String responseString = FileUtils.toString(is); + String responseString = response.getReasonPhrase(); logger.debug("response: {}", responseString); - return response.getStatusLine().getStatusCode(); + return response.getCode(); } } From b143b25115a1394c96f14700a54747ecf72c1196 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sun, 3 Dec 2023 09:29:54 +0100 Subject: [PATCH 02/10] #2296 - disable keepalive strategy to fix SocketException --- .../intuit/karate/http/ApacheHttpClient.java | 60 ++++++++++++------- .../intuit/karate/fatjar/ProxyServerTest.java | 7 --- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 1f7a080ef..1efb2d0f9 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -65,6 +65,7 @@ import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.cookie.MalformedCookieException; import org.apache.hc.client5.http.entity.EntityBuilder; +import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; import org.apache.hc.client5.http.impl.DefaultRedirectStrategy; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; @@ -74,12 +75,14 @@ import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; +import org.apache.hc.client5.http.io.HttpClientConnectionManager; import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.client5.http.ssl.LenientSslConnectionSocketFactory; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; import org.apache.hc.client5.http.ssl.TrustAllStrategy; import org.apache.hc.client5.http.ssl.TrustSelfSignedStrategy; +import org.apache.hc.core5.http.ClassicHttpResponse; import org.apache.hc.core5.http.EntityDetails; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpEntity; @@ -87,15 +90,20 @@ import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpMessage; import org.apache.hc.core5.http.HttpRequestInterceptor; +import org.apache.hc.core5.http.HttpResponse; import org.apache.hc.core5.http.config.Registry; import org.apache.hc.core5.http.config.RegistryBuilder; import org.apache.hc.core5.http.io.SocketConfig; +import org.apache.hc.core5.http.io.entity.EntityUtils; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; import org.apache.hc.core5.http.protocol.HttpContext; import org.apache.hc.core5.net.URIBuilder; +import org.apache.hc.core5.pool.PoolConcurrencyPolicy; +import org.apache.hc.core5.pool.PoolReusePolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.ssl.SSLContexts; import org.apache.http.client.config.AuthSchemes; +import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; /** * @@ -219,7 +227,10 @@ private void configure(Config config) { throw new RuntimeException(e); } } - connectionManagerBuilder.setDefaultConnectionConfig(ConnectionConfig.custom() + connectionManagerBuilder + .setPoolConcurrencyPolicy(PoolConcurrencyPolicy.STRICT) + .setConnPoolPolicy(PoolReusePolicy.LIFO) + .setDefaultConnectionConfig(ConnectionConfig.custom() .setSocketTimeout(config.getReadTimeout(), TimeUnit.MILLISECONDS) .setConnectTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); RequestConfig.Builder configBuilder = RequestConfig.custom() @@ -234,14 +245,18 @@ private void configure(Config config) { clientBuilder.setDefaultCredentialsProvider(credentialsProvider); configBuilder.setTargetPreferredAuthSchemes(authSchemes); } - clientBuilder.setDefaultRequestConfig(configBuilder.build()); - SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS); - connectionManagerBuilder.setDefaultSocketConfig(socketBuilder.build()); - - clientBuilder.setRoutePlanner(buildRoutePlanner(config)); - clientBuilder.setConnectionManager(connectionManagerBuilder.build()); - - clientBuilder.addRequestInterceptorLast(this); + connectionManagerBuilder.setDefaultSocketConfig(SocketConfig.custom() + .setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); + + clientBuilder.setRoutePlanner(buildRoutePlanner(config)) + .setDefaultRequestConfig(configBuilder.build()) + .setConnectionManager(connectionManagerBuilder.build()) + // Not sure about this. With the default reuseStrategy, ProxyServerTest fails with a SocketConnection(client.feature#11). + // Could not work out the exact reason. But the same SocketHandler was being used for the first two calls and was failing the second time. + // By setting a no reuse strategy, the connections are closed and the test passes. + // Impact on performance to be checked. + .setConnectionReuseStrategy((req, resp, ctx) -> false) + .addRequestInterceptorLast(this); } // Differences with httpclient4 implementation: @@ -335,20 +350,16 @@ public Response invoke(HttpRequest request) { } if (request.getHeaders() != null) { request.getHeaders().forEach((k, vals) -> vals.forEach(v -> requestBuilder.addHeader(k, v))); - } - CloseableHttpResponse httpResponse; - byte[] bytes; - try (CloseableHttpClient client = clientBuilder.build()) { - httpResponse = client.execute(requestBuilder.build()); - HttpEntity responseEntity = httpResponse.getEntity(); - if (responseEntity == null || responseEntity.getContent() == null) { - bytes = Constants.ZERO_BYTES; - } else { - InputStream is = responseEntity.getContent(); - bytes = FileUtils.toBytes(is); - } + } + try { + // client can not be closed/autoclosed as it is referenced accross multiple calls by ScenarioEngine + // (requestBuilder.client.invoke(httpRequest)) + CloseableHttpClient client = clientBuilder.build(); + + Response response = client.execute(requestBuilder.build(), this::buildResponse); request.setEndTime(System.currentTimeMillis()); - httpResponse.close(); + httpLogger.logResponse(getConfig(), request, response); + return response; } catch (Exception e) { if (e instanceof ClientProtocolException && e.getCause() != null) { // better error message throw new RuntimeException(e.getCause()); @@ -356,6 +367,11 @@ public Response invoke(HttpRequest request) { throw new RuntimeException(e); } } + } + + private Response buildResponse(ClassicHttpResponse httpResponse) throws IOException{ + HttpEntity entity = httpResponse.getEntity(); + byte[] bytes = entity != null ? EntityUtils.toByteArray(entity) : Constants.ZERO_BYTES; int statusCode = httpResponse.getCode(); Map> headers = toHeaders(httpResponse); List storedCookies = cookieStore.getCookies(); diff --git a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java index 06a1eee2c..43a383187 100644 --- a/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java +++ b/karate-core/src/test/java/com/intuit/karate/fatjar/ProxyServerTest.java @@ -38,14 +38,11 @@ class ProxyServerTest { @BeforeAll static void beforeAll() { proxy = new ProxyServer(0, null, null); - System.out.println("-3"); server = MockServer .feature("classpath:com/intuit/karate/fatjar/server.feature") .pathPrefix("/v1") .http(0).build(); - System.out.println("-2"); int port = server.getPort(); - System.out.println("-1"); System.setProperty("karate.server.port", port + ""); System.setProperty("karate.server.ssl", ""); // for ci System.setProperty("karate.server.proxy", "http://localhost:" + proxy.getPort()); @@ -59,13 +56,9 @@ static void afterAll() { @Test void testProxy() throws Exception { - System.out.println("0"); String url = "http://localhost:" + server.getPort() + "/v1/cats"; - System.out.println("1"); assertEquals(200, http(get(url))); - System.out.println("2"); assertEquals(200, http(post(url, "{ \"name\": \"Billie\" }"))); - System.out.println("3"); Results results = Runner .path("classpath:com/intuit/karate/fatjar/client.feature") .configDir("classpath:com/intuit/karate/fatjar") From b470148967fee331ea448a8e454c64f3d4a68924 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sun, 3 Dec 2023 12:10:42 +0100 Subject: [PATCH 03/10] #2296 - code clean up --- .../main/java/com/intuit/karate/http/ApacheHttpClient.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 1efb2d0f9..90397fe18 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -24,7 +24,6 @@ package com.intuit.karate.http; import com.intuit.karate.Constants; -import com.intuit.karate.FileUtils; import com.intuit.karate.Logger; import com.intuit.karate.core.Config; import com.intuit.karate.core.ScenarioEngine; @@ -32,7 +31,6 @@ import io.netty.handler.codec.http.cookie.ClientCookieDecoder; import io.netty.handler.codec.http.cookie.ServerCookieEncoder; import java.io.IOException; -import java.io.InputStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.Proxy; @@ -65,17 +63,14 @@ import org.apache.hc.client5.http.cookie.CookieStore; import org.apache.hc.client5.http.cookie.MalformedCookieException; import org.apache.hc.client5.http.entity.EntityBuilder; -import org.apache.hc.client5.http.impl.DefaultHttpRequestRetryStrategy; import org.apache.hc.client5.http.impl.DefaultRedirectStrategy; import org.apache.hc.client5.http.impl.auth.BasicCredentialsProvider; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.cookie.CookieSpecBase; import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; -import org.apache.hc.client5.http.io.HttpClientConnectionManager; import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.client5.http.ssl.LenientSslConnectionSocketFactory; import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; @@ -90,7 +85,6 @@ import org.apache.hc.core5.http.HttpHost; import org.apache.hc.core5.http.HttpMessage; import org.apache.hc.core5.http.HttpRequestInterceptor; -import org.apache.hc.core5.http.HttpResponse; import org.apache.hc.core5.http.config.Registry; import org.apache.hc.core5.http.config.RegistryBuilder; import org.apache.hc.core5.http.io.SocketConfig; @@ -103,7 +97,6 @@ import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.ssl.SSLContexts; import org.apache.http.client.config.AuthSchemes; -import org.apache.http.impl.client.DefaultHttpRequestRetryHandler; /** * From abc38b40eab3e2b85d780844a9afe3935fed866a Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sun, 17 Dec 2023 21:06:57 +0100 Subject: [PATCH 04/10] remove references to http client 4 --- karate-core/pom.xml | 18 - .../intuit/karate/http/ApacheHttpClient.java | 7 +- .../intuit/karate/http/ApacheHttpClient4.java | 372 ------------------ .../http/CustomHttpRequestRetryHandler.java | 21 +- .../karate/http/HttpRequestBuilder.java | 2 +- .../LenientSslConnectionSocketFactory.java | 48 --- 6 files changed, 11 insertions(+), 457 deletions(-) delete mode 100644 karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java delete mode 100644 karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java diff --git a/karate-core/pom.xml b/karate-core/pom.xml index 96dd79fea..a43dbb860 100644 --- a/karate-core/pom.xml +++ b/karate-core/pom.xml @@ -49,24 +49,6 @@ - - org.apache.httpcomponents - httpclient - 4.5.14 - - - commons-logging - commons-logging - - - - - - commons-codec - commons-codec - 1.16.0 - - org.apache.httpcomponents.client5 httpclient5 diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 90397fe18..0e69c37f9 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -53,6 +53,7 @@ import org.apache.hc.client5.http.ClientProtocolException; import org.apache.hc.client5.http.auth.AuthScope; import org.apache.hc.client5.http.auth.NTCredentials; +import org.apache.hc.client5.http.auth.StandardAuthScheme; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; @@ -96,7 +97,6 @@ import org.apache.hc.core5.pool.PoolReusePolicy; import org.apache.hc.core5.ssl.SSLContextBuilder; import org.apache.hc.core5.ssl.SSLContexts; -import org.apache.http.client.config.AuthSchemes; /** * @@ -228,9 +228,10 @@ private void configure(Config config) { .setConnectTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); RequestConfig.Builder configBuilder = RequestConfig.custom() .setCookieSpec(LenientCookieSpec.KARATE); - if (config.isNtlmEnabled()) { + if (config.isNtlmEnabled()) { + //Will not be supported as of 5.3. See https://hc.apache.org/httpcomponents-client-5.3.x/current/httpclient5/apidocs/index.html?org/apache/hc/client5/http/auth/NTCredentials.html List authSchemes = new ArrayList<>(); - authSchemes.add(AuthSchemes.NTLM); + authSchemes.add(StandardAuthScheme.NTLM); BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); NTCredentials ntCredentials = new NTCredentials( config.getNtlmUsername(), config.getNtlmPassword().toCharArray(), config.getNtlmWorkstation(), config.getNtlmDomain()); diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java deleted file mode 100644 index 8feb9a572..000000000 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient4.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * The MIT License - * - * Copyright 2022 Karate Labs Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package com.intuit.karate.http; - -import com.intuit.karate.Constants; -import com.intuit.karate.FileUtils; -import com.intuit.karate.Logger; -import com.intuit.karate.core.Config; -import com.intuit.karate.core.ScenarioEngine; -import io.netty.handler.codec.http.cookie.ClientCookieDecoder; -import io.netty.handler.codec.http.cookie.ServerCookieEncoder; -import java.io.IOException; -import java.io.InputStream; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.Socket; -import java.net.SocketAddress; -import java.net.URI; -import java.security.KeyStore; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import javax.net.ssl.SSLContext; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.HttpException; -import org.apache.http.HttpHost; -import org.apache.http.HttpMessage; -import org.apache.http.HttpRequestInterceptor; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.NTCredentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.ClientProtocolException; -import org.apache.http.client.CookieStore; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.config.AuthSchemes; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.EntityBuilder; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.RequestBuilder; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.config.Registry; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.config.SocketConfig; -import org.apache.http.conn.ssl.LenientSslConnectionSocketFactory; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustAllStrategy; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.CookieSpecProvider; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.BasicCredentialsProvider; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.LaxRedirectStrategy; -import org.apache.http.impl.conn.SystemDefaultRoutePlanner; -import org.apache.http.impl.cookie.DefaultCookieSpec; -import org.apache.http.protocol.HttpContext; -import org.apache.http.ssl.SSLContextBuilder; -import org.apache.http.ssl.SSLContexts; - -/** - * - * @author pthomas3 - */ -public class ApacheHttpClient4 implements HttpClient, HttpRequestInterceptor { - - private final ScenarioEngine engine; - private final Logger logger; - private final HttpLogger httpLogger; - - private HttpClientBuilder clientBuilder; - private CookieStore cookieStore; - - public static class LenientCookieSpec extends DefaultCookieSpec { - - static final String KARATE = "karate"; - - public LenientCookieSpec() { - super(new String[]{"EEE, dd-MMM-yy HH:mm:ss z", "EEE, dd MMM yyyy HH:mm:ss Z"}, false); - } - - @Override - public boolean match(Cookie cookie, CookieOrigin origin) { - return true; - } - - @Override - public void validate(Cookie cookie, CookieOrigin origin) throws MalformedCookieException { - // do nothing - } - - public static Registry registry() { - CookieSpecProvider specProvider = (HttpContext hc) -> new LenientCookieSpec(); - return RegistryBuilder.create() - .register(KARATE, specProvider).build(); - } - - } - - public ApacheHttpClient4(ScenarioEngine engine) { - this.engine = engine; - logger = engine.logger; - httpLogger = new HttpLogger(logger); - configure(engine.getConfig()); - } - - private void configure(Config config) { - clientBuilder = HttpClientBuilder.create(); - if (config.isHttpRetryEnabled()) { - clientBuilder.setRetryHandler(new CustomHttpRequestRetryHandler(logger)); - } else { - clientBuilder.disableAutomaticRetries(); - } - - if (!config.isFollowRedirects()) { - clientBuilder.disableRedirectHandling(); - } else { // support redirect on POST by default - clientBuilder.setRedirectStrategy(LaxRedirectStrategy.INSTANCE); - } - cookieStore = new BasicCookieStore(); - clientBuilder.setDefaultCookieStore(cookieStore); - clientBuilder.setDefaultCookieSpecRegistry(LenientCookieSpec.registry()); - clientBuilder.useSystemProperties(); - if (config.isSslEnabled()) { - // System.setProperty("jsse.enableSNIExtension", "false"); - String algorithm = config.getSslAlgorithm(); // could be null - KeyStore trustStore = engine.getKeyStore(config.getSslTrustStore(), config.getSslTrustStorePassword(), config.getSslTrustStoreType()); - KeyStore keyStore = engine.getKeyStore(config.getSslKeyStore(), config.getSslKeyStorePassword(), config.getSslKeyStoreType()); - SSLContext sslContext; - try { - SSLContextBuilder builder = SSLContexts.custom() - .setProtocol(algorithm); // will default to TLS if null - if (trustStore == null && config.isSslTrustAll()) { - builder = builder.loadTrustMaterial(new TrustAllStrategy()); - } else { - if (config.isSslTrustAll()) { - builder = builder.loadTrustMaterial(trustStore, new TrustSelfSignedStrategy()); - } else { - builder = builder.loadTrustMaterial(trustStore, null); // will use system / java default - } - } - if (keyStore != null) { - char[] keyPassword = config.getSslKeyStorePassword() == null ? null : config.getSslKeyStorePassword().toCharArray(); - builder = builder.loadKeyMaterial(keyStore, keyPassword); - } - sslContext = builder.build(); - SSLConnectionSocketFactory socketFactory; - if (keyStore != null) { - socketFactory = new SSLConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); - } else { - socketFactory = new LenientSslConnectionSocketFactory(sslContext, new NoopHostnameVerifier()); - } - clientBuilder.setSSLSocketFactory(socketFactory); - } catch (Exception e) { - logger.error("ssl context init failed: {}", e.getMessage()); - throw new RuntimeException(e); - } - } - RequestConfig.Builder configBuilder = RequestConfig.custom() - .setCookieSpec(LenientCookieSpec.KARATE) - .setConnectTimeout(config.getConnectTimeout()) - .setSocketTimeout(config.getReadTimeout()); - if (config.getLocalAddress() != null) { - try { - InetAddress localAddress = InetAddress.getByName(config.getLocalAddress()); - configBuilder.setLocalAddress(localAddress); - } catch (Exception e) { - logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); - } - } - if (config.isNtlmEnabled()) { - List authSchemes = new ArrayList<>(); - authSchemes.add(AuthSchemes.NTLM); - CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - NTCredentials ntCredentials = new NTCredentials( - config.getNtlmUsername(), config.getNtlmPassword(), config.getNtlmWorkstation(), config.getNtlmDomain()); - credentialsProvider.setCredentials(AuthScope.ANY, ntCredentials); - clientBuilder.setDefaultCredentialsProvider(credentialsProvider); - configBuilder.setTargetPreferredAuthSchemes(authSchemes); - } - clientBuilder.setDefaultRequestConfig(configBuilder.build()); - SocketConfig.Builder socketBuilder = SocketConfig.custom().setSoTimeout(config.getConnectTimeout()); - clientBuilder.setDefaultSocketConfig(socketBuilder.build()); - if (config.getProxyUri() != null) { - try { - URI proxyUri = new URIBuilder(config.getProxyUri()).build(); - clientBuilder.setProxy(new HttpHost(proxyUri.getHost(), proxyUri.getPort(), proxyUri.getScheme())); - if (config.getProxyUsername() != null && config.getProxyPassword() != null) { - CredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials( - new AuthScope(proxyUri.getHost(), proxyUri.getPort()), - new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword())); - clientBuilder.setDefaultCredentialsProvider(credsProvider); - } - if (config.getNonProxyHosts() != null) { - ProxySelector proxySelector = new ProxySelector() { - private final List proxyExceptions = config.getNonProxyHosts(); - - @Override - public List select(URI uri) { - return Collections.singletonList(proxyExceptions.contains(uri.getHost()) - ? Proxy.NO_PROXY - : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); - } - - @Override - public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { - logger.info("connect failed to uri: {}", uri, ioe); - } - }; - clientBuilder.setRoutePlanner(new SystemDefaultRoutePlanner(proxySelector)); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - clientBuilder.addInterceptorLast(this); - } - - @Override - public void setConfig(Config config) { - configure(config); - } - - @Override - public Config getConfig() { - return engine.getConfig(); - } - - private HttpRequest request; - - @Override - public Response invoke(HttpRequest request) { - this.request = request; - RequestBuilder requestBuilder = RequestBuilder.create(request.getMethod()).setUri(request.getUrl()); - if (request.getBody() != null) { - EntityBuilder entityBuilder = EntityBuilder.create().setBinary(request.getBody()); - List transferEncoding = request.getHeaderValues(HttpConstants.HDR_TRANSFER_ENCODING); - if (transferEncoding != null) { - for (String te : transferEncoding) { - if (te == null) { - continue; - } - if (te.contains("chunked")) { // can be comma delimited as per spec - entityBuilder.chunked(); - } - if (te.contains("gzip")) { - entityBuilder.gzipCompress(); - } - } - request.removeHeader(HttpConstants.HDR_TRANSFER_ENCODING); - } - requestBuilder.setEntity(entityBuilder.build()); - } - if (request.getHeaders() != null) { - request.getHeaders().forEach((k, vals) -> vals.forEach(v -> requestBuilder.addHeader(k, v))); - } - CloseableHttpResponse httpResponse; - byte[] bytes; - try (CloseableHttpClient client = clientBuilder.build()) { - httpResponse = client.execute(requestBuilder.build()); - HttpEntity responseEntity = httpResponse.getEntity(); - if (responseEntity == null || responseEntity.getContent() == null) { - bytes = Constants.ZERO_BYTES; - } else { - InputStream is = responseEntity.getContent(); - bytes = FileUtils.toBytes(is); - } - request.setEndTime(System.currentTimeMillis()); - httpResponse.close(); - } catch (Exception e) { - if (e instanceof ClientProtocolException && e.getCause() != null) { // better error message - throw new RuntimeException(e.getCause()); - } else { - throw new RuntimeException(e); - } - } - int statusCode = httpResponse.getStatusLine().getStatusCode(); - Map> headers = toHeaders(httpResponse); - List storedCookies = cookieStore.getCookies(); - Header[] requestCookieHeaders = httpResponse.getHeaders(HttpConstants.HDR_SET_COOKIE); - // edge case where the apache client - // auto-followed a redirect where cookies were involved - List mergedCookieValues = new ArrayList(requestCookieHeaders.length); - Set alreadyMerged = new HashSet(requestCookieHeaders.length); - for (Header ch : requestCookieHeaders) { - String requestCookieValue = ch.getValue(); - io.netty.handler.codec.http.cookie.Cookie c = ClientCookieDecoder.LAX.decode(requestCookieValue); - mergedCookieValues.add(requestCookieValue); - alreadyMerged.add(c.name()); - } - for (Cookie c : storedCookies) { - if (c.getValue() != null) { - String name = c.getName(); - if (alreadyMerged.contains(name)) { - continue; - } - Map map = new HashMap(); - map.put(Cookies.NAME, name); - map.put(Cookies.VALUE, c.getValue()); - map.put(Cookies.DOMAIN, c.getDomain()); - if (c.getExpiryDate() != null) { - map.put(Cookies.MAX_AGE, c.getExpiryDate().getTime()); - } - map.put(Cookies.SECURE, c.isSecure()); - io.netty.handler.codec.http.cookie.Cookie nettyCookie = Cookies.fromMap(map); - String cookieValue = ServerCookieEncoder.LAX.encode(nettyCookie); - mergedCookieValues.add(cookieValue); - } - } - headers.put(HttpConstants.HDR_SET_COOKIE, mergedCookieValues); - cookieStore.clear(); - Response response = new Response(statusCode, headers, bytes); - httpLogger.logResponse(getConfig(), request, response); - return response; - } - - @Override - public void process(org.apache.http.HttpRequest hr, HttpContext hc) throws HttpException, IOException { - request.setHeaders(toHeaders(hr)); - httpLogger.logRequest(getConfig(), request); - request.setStartTime(System.currentTimeMillis()); - } - - private static Map> toHeaders(HttpMessage msg) { - Header[] headers = msg.getAllHeaders(); - Map> map = new LinkedHashMap(headers.length); - for (Header outer : headers) { - String name = outer.getName(); - Header[] inner = msg.getHeaders(name); - List list = new ArrayList(inner.length); - for (Header h : inner) { - list.add(h.getValue()); - } - map.put(name, list); - } - return map; - } - -} diff --git a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java index 6a61af6bf..cf9b81911 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java +++ b/karate-core/src/main/java/com/intuit/karate/http/CustomHttpRequestRetryHandler.java @@ -5,10 +5,9 @@ import org.apache.hc.client5.http.HttpRequestRetryStrategy; import org.apache.hc.core5.http.HttpRequest; import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.NoHttpResponseException; +import org.apache.hc.core5.http.protocol.HttpContext; import org.apache.hc.core5.util.TimeValue; -import org.apache.http.NoHttpResponseException; -import org.apache.http.client.HttpRequestRetryHandler; -import org.apache.http.protocol.HttpContext; import com.intuit.karate.Logger; @@ -17,7 +16,7 @@ * This is usually the case when there is steal connection. The retry cause that * the connection is renewed and the second call will succeed. */ -public class CustomHttpRequestRetryHandler implements HttpRequestRetryHandler, HttpRequestRetryStrategy +public class CustomHttpRequestRetryHandler implements HttpRequestRetryStrategy { private final Logger logger; @@ -26,11 +25,6 @@ public CustomHttpRequestRetryHandler(Logger logger) this.logger = logger; } - @Override - public boolean retryRequest(IOException exception, int executionCount, HttpContext context) { - return shouldRetry(exception, executionCount); - } - private boolean shouldRetry(IOException exception, int executionCount) { if (exception instanceof NoHttpResponseException && executionCount < 1) { @@ -45,20 +39,17 @@ private boolean shouldRetry(IOException exception, int executionCount) { } @Override - public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, - org.apache.hc.core5.http.protocol.HttpContext context) { + public boolean retryRequest(HttpRequest request, IOException exception, int executionCount, HttpContext context) { return shouldRetry(exception, executionCount); } @Override - public boolean retryRequest(HttpResponse response, int execCount, - org.apache.hc.core5.http.protocol.HttpContext context) { + public boolean retryRequest(HttpResponse response, int execCount, HttpContext context) { return false; } @Override - public TimeValue getRetryInterval(HttpResponse response, int execCount, - org.apache.hc.core5.http.protocol.HttpContext context) { + public TimeValue getRetryInterval(HttpResponse response, int execCount, HttpContext context) { return TimeValue.ofSeconds(1); // NOt sure what the interval was in httpclient4 ... Sticking with the default value of the default http5 implementation. } } \ No newline at end of file diff --git a/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java b/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java index 98f6937ac..f8989d1c3 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java +++ b/karate-core/src/main/java/com/intuit/karate/http/HttpRequestBuilder.java @@ -51,7 +51,7 @@ import java.util.function.Supplier; import java.util.stream.Collectors; -import org.apache.http.client.utils.URIBuilder; +import org.apache.hc.core5.net.URIBuilder; import org.graalvm.polyglot.Value; import org.graalvm.polyglot.proxy.ProxyObject; import org.slf4j.Logger; diff --git a/karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java b/karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java deleted file mode 100644 index 0441f6a2d..000000000 --- a/karate-core/src/main/java/org/apache/http/conn/ssl/LenientSslConnectionSocketFactory.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License - * - * Copyright 2022 Karate Labs Inc. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ -package org.apache.http.conn.ssl; - -import java.io.IOException; -import java.net.Socket; -import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; -import org.apache.http.protocol.HttpContext; - -/** - * in a separate package just for log level config consistency - * - * @author pthomas3 - */ -public class LenientSslConnectionSocketFactory extends SSLConnectionSocketFactory { - - public LenientSslConnectionSocketFactory(SSLContext sslContext, HostnameVerifier hostnameVerifier) { - super(sslContext, hostnameVerifier); - } - - @Override - public Socket createLayeredSocket(Socket socket, String target, int port, HttpContext context) throws IOException { - return super.createLayeredSocket(socket, "", port, context); - } - -} From 683ce85a460d7d1365982b6d3c53aca48e7eb6e9 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Tue, 19 Dec 2023 21:42:18 +0100 Subject: [PATCH 05/10] #2296 Upgraded to latest apache httpclient (5.3) and dropped support of ntlm --- karate-core/pom.xml | 2 +- .../java/com/intuit/karate/http/ApacheHttpClient.java | 11 ++--------- .../com/intuit/karate/core/FeatureRuntimeTest.java | 9 +++++---- .../intuit/karate/core/ntlm-authentication.feature | 10 ---------- 4 files changed, 8 insertions(+), 24 deletions(-) delete mode 100644 karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature diff --git a/karate-core/pom.xml b/karate-core/pom.xml index a43dbb860..7f0c02a0c 100644 --- a/karate-core/pom.xml +++ b/karate-core/pom.xml @@ -52,7 +52,7 @@ org.apache.httpcomponents.client5 httpclient5 - 5.2.1 + 5.3 ch.qos.logback diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 0e69c37f9..37ab32824 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -229,15 +229,8 @@ private void configure(Config config) { RequestConfig.Builder configBuilder = RequestConfig.custom() .setCookieSpec(LenientCookieSpec.KARATE); if (config.isNtlmEnabled()) { - //Will not be supported as of 5.3. See https://hc.apache.org/httpcomponents-client-5.3.x/current/httpclient5/apidocs/index.html?org/apache/hc/client5/http/auth/NTCredentials.html - List authSchemes = new ArrayList<>(); - authSchemes.add(StandardAuthScheme.NTLM); - BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - NTCredentials ntCredentials = new NTCredentials( - config.getNtlmUsername(), config.getNtlmPassword().toCharArray(), config.getNtlmWorkstation(), config.getNtlmDomain()); - credentialsProvider.setCredentials(ANY_AUTH_SCOPE, ntCredentials); - clientBuilder.setDefaultCredentialsProvider(credentialsProvider); - configBuilder.setTargetPreferredAuthSchemes(authSchemes); + //No longer supported since 5.3. See https://hc.apache.org/httpcomponents-client-5.3.x/current/httpclient5/apidocs/index.html?org/apache/hc/client5/http/auth/NTCredentials.html + throw new UnsupportedOperationException("NTLM is not supported any more. Please consider using Basic or Bearer authentication with TLS instead."); } connectionManagerBuilder.setDefaultSocketConfig(SocketConfig.custom() .setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); diff --git a/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java b/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java index a09eebd90..4c2af4185 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java +++ b/karate-core/src/test/java/com/intuit/karate/core/FeatureRuntimeTest.java @@ -401,9 +401,10 @@ void testTypeConv() { run("type-conv.feature"); } - @Test - void testConfigureNtlmAuthentication() { - run("ntlm-authentication.feature"); - } + // NTLM not supported in apache client 5.3 + // @Test + // void testConfigureNtlmAuthentication() { + // run("ntlm-authentication.feature"); + // } } diff --git a/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature b/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature deleted file mode 100644 index c89cc701e..000000000 --- a/karate-core/src/test/java/com/intuit/karate/core/ntlm-authentication.feature +++ /dev/null @@ -1,10 +0,0 @@ -Feature: ntlm authentication - - Scenario: various ways to configure ntlm authentication - * configure ntlmAuth = { username: 'admin', password: 'secret', domain: 'my.domain', workstation: 'my-pc' } - * configure ntlmAuth = { username: 'admin', password: 'secret' } - * configure ntlmAuth = null - * eval - """ - karate.configure('ntlmAuth', { username: 'admin', password: 'secret' }) - """ From d9d58e92ba9949041bb080835d9cf1c11d2ace4e Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sat, 23 Dec 2023 14:08:57 +0100 Subject: [PATCH 06/10] 2296 Cosmetic change --- .../src/main/java/com/intuit/karate/http/ApacheHttpClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 37ab32824..337d27fc1 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -230,7 +230,7 @@ private void configure(Config config) { .setCookieSpec(LenientCookieSpec.KARATE); if (config.isNtlmEnabled()) { //No longer supported since 5.3. See https://hc.apache.org/httpcomponents-client-5.3.x/current/httpclient5/apidocs/index.html?org/apache/hc/client5/http/auth/NTCredentials.html - throw new UnsupportedOperationException("NTLM is not supported any more. Please consider using Basic or Bearer authentication with TLS instead."); + throw new UnsupportedOperationException("NTLM authentication is not supported any more. Please consider using Basic or Bearer authentication with TLS instead."); } connectionManagerBuilder.setDefaultSocketConfig(SocketConfig.custom() .setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); From 715ed4ae5f396ca3b70bb05a4f41e75ef8676d59 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Sat, 23 Dec 2023 17:27:19 +0100 Subject: [PATCH 07/10] Upgrade karate-demo to same version of httpclient5 as karate-core --- karate-demo/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/karate-demo/pom.xml b/karate-demo/pom.xml index 40bb062a6..e474f2421 100644 --- a/karate-demo/pom.xml +++ b/karate-demo/pom.xml @@ -20,6 +20,12 @@ + + org.apache.httpcomponents.client5 + httpclient5 + + 5.3 + org.springframework.boot spring-boot-dependencies From 7246fa5fb3c428088166d52e8bc7c016c839a2a6 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Wed, 10 Jan 2024 20:32:43 +0100 Subject: [PATCH 08/10] Introduced close() method in HttpClient and use it to close the connectionManager in Apache client which is now shared --- .../intuit/karate/http/ApacheHttpClient.java | 21 ++++++++++++------- .../intuit/karate/http/ArmeriaHttpClient.java | 5 +++++ .../com/intuit/karate/http/HttpClient.java | 1 + .../com/intuit/karate/core/DummyClient.java | 5 +++++ .../com/intuit/karate/core/MockClient.java | 5 +++++ 5 files changed, 29 insertions(+), 8 deletions(-) diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index 337d27fc1..d290280f1 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -70,6 +70,7 @@ import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; import org.apache.hc.client5.http.impl.cookie.CookieSpecBase; import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; import org.apache.hc.client5.http.routing.HttpRoutePlanner; @@ -104,8 +105,6 @@ */ public class ApacheHttpClient implements HttpClient, HttpRequestInterceptor { - private static final AuthScope ANY_AUTH_SCOPE = new AuthScope(null, null, -1, null, null); - private final ScenarioEngine engine; private final Logger logger; private final HttpLogger httpLogger; @@ -235,9 +234,14 @@ private void configure(Config config) { connectionManagerBuilder.setDefaultSocketConfig(SocketConfig.custom() .setSoTimeout(config.getConnectTimeout(), TimeUnit.MILLISECONDS).build()); + connManager = connectionManagerBuilder.build(); clientBuilder.setRoutePlanner(buildRoutePlanner(config)) .setDefaultRequestConfig(configBuilder.build()) - .setConnectionManager(connectionManagerBuilder.build()) + // set shared flag to true so that we can close the client. + //ConnectionManager won't be closed automatically by Apache, it is now our responsability to do so. + // See comments in https://github.com/karatelabs/karate/pull/2471 + .setConnectionManagerShared(true) + .setConnectionManager(connManager) // Not sure about this. With the default reuseStrategy, ProxyServerTest fails with a SocketConnection(client.feature#11). // Could not work out the exact reason. But the same SocketHandler was being used for the first two calls and was failing the second time. // By setting a no reuse strategy, the connections are closed and the test passes. @@ -311,6 +315,7 @@ public Config getConfig() { } private HttpRequest request; + private PoolingHttpClientConnectionManager connManager; @Override public Response invoke(HttpRequest request) { @@ -338,11 +343,7 @@ public Response invoke(HttpRequest request) { if (request.getHeaders() != null) { request.getHeaders().forEach((k, vals) -> vals.forEach(v -> requestBuilder.addHeader(k, v))); } - try { - // client can not be closed/autoclosed as it is referenced accross multiple calls by ScenarioEngine - // (requestBuilder.client.invoke(httpRequest)) - CloseableHttpClient client = clientBuilder.build(); - + try (CloseableHttpClient client = clientBuilder.build()) { Response response = client.execute(requestBuilder.build(), this::buildResponse); request.setEndTime(System.currentTimeMillis()); httpLogger.logResponse(getConfig(), request, response); @@ -421,6 +422,10 @@ private static Map> toHeaders(HttpMessage msg) { return map; } + public void close() { + connManager.close(); + } + private static class CustomRoutePlanner extends SystemDefaultRoutePlanner { private final InetAddress localAddress; diff --git a/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java index 174c54057..738fb0ae1 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ArmeriaHttpClient.java @@ -134,4 +134,9 @@ public HttpResponse execute(com.linecorp.armeria.client.HttpClient delegate, Cli return delegate.execute(ctx, req); } + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 + } + } diff --git a/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java index c6f19dfa3..f96a96353 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/HttpClient.java @@ -37,4 +37,5 @@ public interface HttpClient { Response invoke(HttpRequest request); + void close(); } diff --git a/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java b/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java index e3bcac9d2..eeb67757e 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java +++ b/karate-core/src/test/java/com/intuit/karate/core/DummyClient.java @@ -24,6 +24,11 @@ public Config getConfig() { @Override public Response invoke(HttpRequest request) { throw new UnsupportedOperationException("not implemented"); + } + + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 } } diff --git a/karate-core/src/test/java/com/intuit/karate/core/MockClient.java b/karate-core/src/test/java/com/intuit/karate/core/MockClient.java index fbef24633..1b3cedaed 100644 --- a/karate-core/src/test/java/com/intuit/karate/core/MockClient.java +++ b/karate-core/src/test/java/com/intuit/karate/core/MockClient.java @@ -33,4 +33,9 @@ public Response invoke(HttpRequest request) { return handler.handle(request.toRequest()); } + @Override + public void close() { + // No op. close() was introduced mainly for ApacheHttpClient, see https://github.com/karatelabs/karate/pull/2471 + } + } From f9a530bd6bc5855e25aff39b96e6b02916ac8d84 Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Tue, 23 Jan 2024 20:57:24 +0100 Subject: [PATCH 09/10] Redesigned RoutePlanner - Separate ProxySelector use cases from proxy/noproxy ones --- karate-core/pom.xml | 7 + .../intuit/karate/http/ApacheHttpClient.java | 148 +++++++++++------- .../karate/http/ApacheHttpServerTest.java | 91 +++++++++++ 3 files changed, 187 insertions(+), 59 deletions(-) create mode 100644 karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java diff --git a/karate-core/pom.xml b/karate-core/pom.xml index 7f0c02a0c..287965631 100644 --- a/karate-core/pom.xml +++ b/karate-core/pom.xml @@ -123,6 +123,13 @@ ${junit5.version} test + + org.mockito + mockito-core + 5.5.0 + test + + diff --git a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java index d290280f1..2d90d4e1a 100644 --- a/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java +++ b/karate-core/src/main/java/com/intuit/karate/http/ApacheHttpClient.java @@ -37,6 +37,7 @@ import java.net.ProxySelector; import java.net.SocketAddress; import java.net.URI; +import java.net.URISyntaxException; import java.security.KeyStore; import java.util.ArrayList; import java.util.Collections; @@ -52,8 +53,6 @@ import org.apache.hc.client5.http.ClientProtocolException; import org.apache.hc.client5.http.auth.AuthScope; -import org.apache.hc.client5.http.auth.NTCredentials; -import org.apache.hc.client5.http.auth.StandardAuthScheme; import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; @@ -72,6 +71,7 @@ import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManagerBuilder; +import org.apache.hc.client5.http.impl.routing.DefaultRoutePlanner; import org.apache.hc.client5.http.impl.routing.SystemDefaultRoutePlanner; import org.apache.hc.client5.http.routing.HttpRoutePlanner; import org.apache.hc.client5.http.ssl.LenientSslConnectionSocketFactory; @@ -252,56 +252,70 @@ private void configure(Config config) { // Differences with httpclient4 implementation: // - RequestBuilder.setLocalAddress does not exist any more, so instead, RoutePlanner.determineLocalAddress is overridden - // - clientBuilder.setProxy is not set any more. I'm probably misreading the code, but looking at DefaultRoutePanner.determineRoute, if proxy exists, my understanding is that determineProxy is not called and therefore proxySelector will NOT be used. - // so ProxySelector has been redesigned to handle both the specified proxy, and the nonProxyhosts if specified. - // Note that the route planner must now handle: - // - localaddress - // - proxy (set or not) - // - nonProxy hosts (set or not) - // Only SystemDefaultRoutePlanner supports proxy/nonProxy host so we subclass that class. However, SystemDefaultRoutePlanner does not have a "no proxy" mode so determineProxy is overridden to opt out if needed. - private HttpRoutePlanner buildRoutePlanner(Config config) { - ProxySelector proxySelector = null; + // - clientBuilder.setProxy does not exist any more. + // Instead, the new RoutePlanner exposes determineProxy and determineLocalAddress methods that may be overridden. + // Karate actually uses two flavors of RoutePlanner's which both implement those methods: + // - one that leverages ProxySelector when the nonProxyHosts property is specified + // - a default one in all other cases, whether a proxy is specified or not. + + protected HttpRoutePlanner buildRoutePlanner(Config config) { + + // Handle localAddress. + // From a Karate perspective, localAddress is primarily designed to be used with Gatling and is not related to proxy. + // However, in apache client 5, it is handled by the RoutePlanner too. + InetAddress localAddress = null; + if (config.getLocalAddress() != null) { + try { + localAddress = InetAddress.getByName(config.getLocalAddress()); + } catch (Exception e) { + logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); + } + } + HttpHost proxy; if (config.getProxyUri() != null) { + URI proxyUri; try { - URI proxyUri = new URIBuilder(config.getProxyUri()).build(); - - proxySelector = new ProxySelector() { - private final List proxyExceptions = config.getNonProxyHosts() == null ? Collections.emptyList() : config.getNonProxyHosts(); + proxyUri = new URIBuilder(config.getProxyUri()).build(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + // Manage proxy authenticator. + // Unfortunately, default credentials are part of the clientBuilder, not routePlanner, so there's a side effect on clientBuilder here. + if (config.getProxyUsername() != null && config.getProxyPassword() != null) { + BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(proxyUri.getHost(), proxyUri.getPort()), + new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword().toCharArray())); + clientBuilder.setDefaultCredentialsProvider(credsProvider); + } + if (config.getNonProxyHosts() != null) { + // Create ProxySelector and its associated route planner + ProxySelector proxySelector = new ProxySelector() { + @Override public List select(URI uri) { - return Collections.singletonList(proxyExceptions.contains(uri.getHost()) + return Collections.singletonList(proxyUri == null || config.getNonProxyHosts().contains(uri.getHost()) ? Proxy.NO_PROXY : new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyUri.getHost(), proxyUri.getPort()))); } - + @Override public void connectFailed(URI uri, SocketAddress sa, IOException ioe) { logger.info("connect failed to uri: {}", uri, ioe); } }; - - if (config.getProxyUsername() != null && config.getProxyPassword() != null) { - BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); - credsProvider.setCredentials( - new AuthScope(proxyUri.getHost(), proxyUri.getPort()), - new UsernamePasswordCredentials(config.getProxyUsername(), config.getProxyPassword().toCharArray())); - clientBuilder.setDefaultCredentialsProvider(credsProvider); - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - InetAddress localAddress = null; - if (config.getLocalAddress() != null) { - try { - localAddress = InetAddress.getByName(config.getLocalAddress()); - } catch (Exception e) { - logger.warn("failed to resolve local address: {} - {}", config.getLocalAddress(), e.getMessage()); + return new ProxySelectorRoutePlanner(proxySelector, localAddress); + } else { + // use simple proxy + proxy = new HttpHost(proxyUri.getScheme(), proxyUri.getHost(), proxyUri.getPort()); } + } else { + // NO proxy at all + proxy = null; } - - return new CustomRoutePlanner(proxySelector, localAddress); + return new ProxyableRoutePlanner(proxy, localAddress); } @Override @@ -426,37 +440,53 @@ public void close() { connManager.close(); } - private static class CustomRoutePlanner extends SystemDefaultRoutePlanner { + /** + * Extends SystemDefaultRoutePlanner to add support for localAddress. + * To be used when nonProxyHosts are specified + */ + private static class ProxySelectorRoutePlanner extends SystemDefaultRoutePlanner { private final InetAddress localAddress; - private ProxySelector proxySelector; - public CustomRoutePlanner(ProxySelector proxySelector, InetAddress localAddress) { + public ProxySelectorRoutePlanner(ProxySelector proxySelector, InetAddress localAddress) { super(proxySelector); - // ProxySelector in SystemDefaultRoutePlanner is private; - this.proxySelector = proxySelector; this.localAddress = localAddress; } - - @Override - protected HttpHost determineProxy( - final HttpHost target, - final HttpContext context) throws HttpException { - if (proxySelector == null) { - //SystemDefaultRoutePlanner will default to some system-wide proxySelector. - //However, the expected behavior here is to ignore proxy altogether if no selector is supplied. - return null; - } - return super.determineProxy(target, context); - } - protected InetAddress determineLocalAddress( - final HttpHost firstHop, - final HttpContext context) throws HttpException { - return localAddress; - } + protected InetAddress determineLocalAddress( + final HttpHost firstHop, + final HttpContext context) throws HttpException { + return localAddress; + } + } + + /** + * Default Route planner that supports localAddress. + * May be used with or without a Proxy, but not with a ProxySelector. + */ + private static class ProxyableRoutePlanner extends DefaultRoutePlanner { + private HttpHost proxy; + private InetAddress localAddress; + public ProxyableRoutePlanner(HttpHost proxy, InetAddress localAddress) { + super(null); + this.proxy = proxy; + this.localAddress = localAddress; + } + @Override + protected HttpHost determineProxy( + final HttpHost target, + final HttpContext context) throws HttpException { + return proxy; + } + + @Override + protected InetAddress determineLocalAddress( + final HttpHost firstHop, + final HttpContext context) throws HttpException { + return localAddress; + } } } diff --git a/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java new file mode 100644 index 000000000..989a3e9ff --- /dev/null +++ b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java @@ -0,0 +1,91 @@ +package com.intuit.karate.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import org.apache.hc.client5.http.HttpRoute; +import org.apache.hc.core5.http.HttpException; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.intuit.karate.core.Config; +import com.intuit.karate.core.ScenarioEngine; +import com.intuit.karate.core.Variable; + +class ApacheHttpServerTest { + + private ScenarioEngine engine; + private Config config; + private HttpHost host; + private HttpContext context; + + private ApacheHttpClient client; + + @BeforeEach + void configure() { + engine = mock(ScenarioEngine.class); + config = new Config(); + Mockito.when(engine.getConfig()).thenReturn(config); + host = new HttpHost("foo.com"); + context = mock(HttpContext.class); + + client = new ApacheHttpClient(engine); + } + + @Test + void noProxy() { + HttpRoute route = determineRoute(client, host); + Assertions.assertNull(route.getProxyHost()); + assertNull(route.getLocalAddress()); + } + + @Test + void proxy() { + config.configure("proxy", new Variable("http://proxy:80")); + HttpRoute route = determineRoute(client, host); + assertEquals("http://proxy:80", route.getProxyHost().toURI()); + } + + @Test + void nonProxyHosts() { + Map proxyConfiguration = new HashMap<>(); + proxyConfiguration.put("uri", "http://proxy:80"); + proxyConfiguration.put("nonProxyHosts", Collections.singletonList("foo.com")); + config.configure("proxy", new Variable(proxyConfiguration)); + + HttpRoute nonProxiedRoute = determineRoute(client, host); + assertNull(nonProxiedRoute.getProxyHost()); + + HttpRoute proxiedRoute = determineRoute(client, new HttpHost("bar.com")); + assertEquals("http://proxy:80", proxiedRoute.getProxyHost().toURI()); + } + + // From a Karate perspective, localAddress is primarily designed to be used with Gatling and is not related to proxy. + // However, in apache client 5, it is handled by the RoutePlanner. + @Test + void localAddress() { + config.configure("localAddress", new Variable("localhost")); + + HttpRoute route = determineRoute(client, host); + + assertNull(route.getProxyHost()); + assertEquals("localhost", route.getLocalAddress().getHostName()); + } + + private HttpRoute determineRoute(ApacheHttpClient client, HttpHost host) { + try { + return client.buildRoutePlanner(config).determineRoute(host, context); + } catch (HttpException e) { + throw new RuntimeException(e); + } + } +} From b9507de3d4b2d8460526b1804150976c9c428fac Mon Sep 17 00:00:00 2001 From: f-delahaye Date: Wed, 24 Jan 2024 18:12:42 +0100 Subject: [PATCH 10/10] cosmetic changes --- .../com/intuit/karate/http/ApacheHttpServerTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java index 989a3e9ff..1b8991aaa 100644 --- a/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java +++ b/karate-core/src/test/java/com/intuit/karate/http/ApacheHttpServerTest.java @@ -43,7 +43,7 @@ void configure() { @Test void noProxy() { - HttpRoute route = determineRoute(client, host); + HttpRoute route = determineRoute(host); Assertions.assertNull(route.getProxyHost()); assertNull(route.getLocalAddress()); } @@ -51,7 +51,7 @@ void noProxy() { @Test void proxy() { config.configure("proxy", new Variable("http://proxy:80")); - HttpRoute route = determineRoute(client, host); + HttpRoute route = determineRoute(host); assertEquals("http://proxy:80", route.getProxyHost().toURI()); } @@ -62,10 +62,10 @@ void nonProxyHosts() { proxyConfiguration.put("nonProxyHosts", Collections.singletonList("foo.com")); config.configure("proxy", new Variable(proxyConfiguration)); - HttpRoute nonProxiedRoute = determineRoute(client, host); + HttpRoute nonProxiedRoute = determineRoute(host); assertNull(nonProxiedRoute.getProxyHost()); - HttpRoute proxiedRoute = determineRoute(client, new HttpHost("bar.com")); + HttpRoute proxiedRoute = determineRoute(new HttpHost("bar.com")); assertEquals("http://proxy:80", proxiedRoute.getProxyHost().toURI()); } @@ -75,13 +75,13 @@ void nonProxyHosts() { void localAddress() { config.configure("localAddress", new Variable("localhost")); - HttpRoute route = determineRoute(client, host); + HttpRoute route = determineRoute(host); assertNull(route.getProxyHost()); assertEquals("localhost", route.getLocalAddress().getHostName()); } - private HttpRoute determineRoute(ApacheHttpClient client, HttpHost host) { + private HttpRoute determineRoute(HttpHost host) { try { return client.buildRoutePlanner(config).determineRoute(host, context); } catch (HttpException e) {