From e5330b132b0d855e1ea5ba9e6f35ffe90d4f16bf Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 19:43:37 +0800 Subject: [PATCH 1/8] browser-like user-agent --- .../clickhouse/client/ClickHouseConfig.java | 61 +++++++++++ .../client/config/ClickHouseClientOption.java | 101 +++++++++++++++++- .../client/config/ClickHouseOption.java | 15 ++- .../client/ClickHouseConfigTest.java | 27 +++++ .../grpc/ClickHouseGrpcChannelFactory.java | 13 ++- .../client/grpc/NettyChannelFactoryImpl.java | 8 ++ .../client/grpc/OkHttpChannelFactoryImpl.java | 8 ++ .../client/http/ApacheHttpConnectionImpl.java | 32 ++++-- .../client/http/ClickHouseHttpClient.java | 3 +- .../client/http/ClickHouseHttpConnection.java | 23 +++- .../client/http/HttpUrlConnectionImpl.java | 7 ++ .../client/http/HttpClientConnectionImpl.java | 7 ++ .../client/http/ClickHouseHttpClientTest.java | 60 +++++++++++ .../http/ClickHouseHttpConnectionTest.java | 4 +- .../com/clickhouse/jdbc/ClickHouseDriver.java | 2 +- .../internal/ClickHouseJdbcUrlParser.java | 1 + .../connection/ClickHouseConnection.java | 6 +- 17 files changed, 356 insertions(+), 22 deletions(-) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java index 106347526..e2383f16c 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseConfig.java @@ -149,6 +149,7 @@ public static Map toClientOptions(Map prop } String key = e.getKey().toString(); + // no need to enable option overidding for now ClickHouseOption o = ClickHouseClientOption.fromKey(key); if (o == null) { o = customOptions.get(key); @@ -190,6 +191,7 @@ public static Map toClientOptions(Map prop private final int maxQueuedRequests; private final long maxResultRows; private final int maxThreads; + private final String productName; private final int nodeCheckInterval; private final int failover; private final int retry; @@ -297,6 +299,7 @@ public ClickHouseConfig(Map options, ClickHouseC this.maxQueuedRequests = getIntOption(ClickHouseClientOption.MAX_QUEUED_REQUESTS); this.maxResultRows = getLongOption(ClickHouseClientOption.MAX_RESULT_ROWS); this.maxThreads = getIntOption(ClickHouseClientOption.MAX_THREADS_PER_CLIENT); + this.productName = getStrOption(ClickHouseClientOption.PRODUCT_NAME); this.nodeCheckInterval = getIntOption(ClickHouseClientOption.NODE_CHECK_INTERVAL); this.failover = getIntOption(ClickHouseClientOption.FAILOVER); this.retry = getIntOption(ClickHouseClientOption.RETRY); @@ -525,6 +528,10 @@ public int getMaxThreadsPerClient() { return maxThreads; } + public String getProductName() { + return productName; + } + public int getFailover() { return failover; } @@ -642,6 +649,60 @@ public TimeZone getUseTimeZone() { return useTimeZone; } + /** + * Same as {@link ClickHouseClientOption#PRODUCT_VERSION}. + * + * @return non-empty semantic version + */ + public final String getProductVersion() { + return ClickHouseClientOption.PRODUCT_VERSION; + } + + /** + * Same as {@link ClickHouseClientOption#PRODUCT_REVISION}. + * + * @return non-empty revision + */ + public final String getProductRevision() { + return ClickHouseClientOption.PRODUCT_REVISION; + } + + /** + * Same as {@link ClickHouseClientOption#CLIENT_OS_INFO}. + * + * @return non-empty O/S information + */ + public final String getClientOsInfo() { + return ClickHouseClientOption.CLIENT_OS_INFO; + } + + /** + * Same as {@link ClickHouseClientOption#CLIENT_JVM_INFO}. + * + * @return non-empty JVM information + */ + public final String getClientJvmInfo() { + return ClickHouseClientOption.CLIENT_JVM_INFO; + } + + /** + * Same as {@link ClickHouseClientOption#CLIENT_USER}. + * + * @return non-empty user name + */ + public final String getClientUser() { + return ClickHouseClientOption.CLIENT_USER; + } + + /** + * Same as {@link ClickHouseClientOption#CLIENT_HOST}. + * + * @return non-empty host name + */ + public final String getClientHost() { + return ClickHouseClientOption.CLIENT_HOST; + } + public ClickHouseCredentials getDefaultCredentials() { return this.credentials; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java index 57cb64654..56f4308e4 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java @@ -1,6 +1,8 @@ package com.clickhouse.client.config; import java.io.Serializable; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -115,7 +117,7 @@ public enum ClickHouseClientOption implements ClickHouseOption { /** * Client name. */ - CLIENT_NAME("client_name", "ClickHouse Java Client", + CLIENT_NAME("client_name", DEFAULT_CLIENT_NAME, "Client name, which is either 'client_name' or 'http_user_agent' shows up in system.query_log table."), /** * Whether server will compress response to client or not. @@ -201,6 +203,10 @@ public enum ClickHouseClientOption implements ClickHouseOption { */ MAX_THREADS_PER_CLIENT("max_threads_per_client", 0, "Size of thread pool for each client instance, 0 or negative number means the client will use shared thread pool."), + /** + * Product name usered in user agent. + */ + PRODUCT_NAME("product_name", DEFAULT_PRODUCT_NAME, "Product name used in user agent."), /** * Method to rename response columns. */ @@ -376,16 +382,105 @@ public enum ClickHouseClientOption implements ClickHouseOption { private static final Map options; + static final String UNKNOWN = "unknown"; + + /** + * Semantic version of the product. + */ + public static final String PRODUCT_VERSION; + /** + * Revision(shortened git commit hash) of the product. + */ + public static final String PRODUCT_REVISION; + /** + * Client O/S information in format of {@code /}. + */ + public static final String CLIENT_OS_INFO; + /** + * Client JVM information in format of {@code /}. + */ + public static final String CLIENT_JVM_INFO; + /** + * Client user name. + */ + public static final String CLIENT_USER; + /** + * Client host name. + */ + public static final String CLIENT_HOST; + static { Map map = new HashMap<>(); - for (ClickHouseClientOption o : values()) { if (map.put(o.getKey(), o) != null) { throw new IllegalStateException("Duplicated key found: " + o.getKey()); } } - options = Collections.unmodifiableMap(map); + + // (revision: ) + String ver = ClickHouseClientOption.class.getPackage().getImplementationVersion(); + String[] parts = ver == null || ver.isEmpty() ? null : ver.split("\\s"); + if (parts != null && parts.length == 4 && parts[1].length() > 0 && parts[3].length() > 1 + && ver.charAt(ver.length() - 1) == ')') { + PRODUCT_VERSION = parts[1]; + ver = parts[3]; + PRODUCT_REVISION = ver.substring(0, ver.length() - 1); + } else { // perhaps try harder by checking version from pom.xml? + PRODUCT_VERSION = UNKNOWN; + PRODUCT_REVISION = UNKNOWN; + } + CLIENT_OS_INFO = new StringBuilder().append(getSystemConfig("os.name", "O/S")).append('/') + .append(getSystemConfig("os.version", UNKNOWN)).toString(); + String javaVersion = System.getProperty("java.vendor.version"); + if (javaVersion == null || javaVersion.isEmpty() || javaVersion.indexOf(' ') >= 0) { + javaVersion = getSystemConfig("java.vm.version", getSystemConfig("java.version", UNKNOWN)); + } + CLIENT_JVM_INFO = new StringBuilder().append(getSystemConfig("java.vm.name", "Java")).append('/') + .append(javaVersion).toString(); + CLIENT_USER = getSystemConfig("user.name", UNKNOWN); + try { + ver = InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e1) { + // ignore + } + CLIENT_HOST = ver == null || ver.isEmpty() ? UNKNOWN : ver; + } + + /** + * Builds user-agent based on given product name. The user-agent will be + * something look like + * {@code / (/; /[; ]; rv:)}. + * + * @param productName product name, null or blank string is treated as + * {@code ClickHouse Java Client} + * @param additionalProperty additional property if any + * @return non-empty user-agent + */ + public static final String buildUserAgent(String productName, String additionalProperty) { + productName = productName == null || productName.isEmpty() ? DEFAULT_PRODUCT_NAME : productName.trim(); + StringBuilder builder = new StringBuilder(productName).append('/').append(PRODUCT_VERSION).append(" (") + .append(CLIENT_OS_INFO).append("; ").append(CLIENT_JVM_INFO); + if (additionalProperty != null && !additionalProperty.isEmpty()) { + builder.append("; ").append(additionalProperty.trim()); + } + return builder.append("; rv:").append(PRODUCT_REVISION).append(')').toString(); + } + + /** + * Gets system property and fall back to the given value as needed. + * + * @param propertyName non-null property name + * @param defaultValue default value, only useful if it's not null + * @return property value + */ + public static String getSystemConfig(String propertyName, String defaultValue) { + final String propertyValue = System.getProperty(propertyName); + if (defaultValue == null) { + return propertyValue; + } + + return propertyValue == null || propertyValue.isEmpty() ? defaultValue : propertyValue; } /** diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java index 31e7e02d5..712656fae 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java @@ -16,13 +16,22 @@ * description. */ public interface ClickHouseOption extends Serializable { + /** + * Default client name. + */ + static final String DEFAULT_CLIENT_NAME = "ClickHouse Java Client"; + /** + * Default product name. + */ + static final String DEFAULT_PRODUCT_NAME = "ClickHouse-JavaClient"; + /** * Converts given string to key value pairs. * * @param str string * @return non-null key value pairs */ - public static Map toKeyValuePairs(String str) { + static Map toKeyValuePairs(String str) { if (str == null || str.isEmpty()) { return Collections.emptyMap(); } @@ -76,7 +85,7 @@ public static Map toKeyValuePairs(String str) { * @return non-null typed value */ @SuppressWarnings("unchecked") - public static T fromString(String value, Class clazz) { + static T fromString(String value, Class clazz) { if (clazz == null) { throw new IllegalArgumentException("Non-null value type is required"); } else if (value == null) { @@ -143,7 +152,7 @@ public static T fromString(String value, Class clazz * @return non-null typed value */ @SuppressWarnings("unchecked") - public static T fromString(String value, T defaultValue) { + static T fromString(String value, T defaultValue) { if (defaultValue == null) { throw new IllegalArgumentException("Non-null default value is required"); } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java index c213dc674..1f994457f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java @@ -1,6 +1,9 @@ package com.clickhouse.client; import java.io.Serializable; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import org.testng.Assert; @@ -66,4 +69,28 @@ public void testCustomValues() { Assert.assertEquals(config.getPreferredTags().size(), 0); Assert.assertEquals(config.getMetricRegistry().get(), metricRegistry); } + + @Test(groups = { "unit" }) + public void testClientInfo() throws UnknownHostException { + ClickHouseConfig config = new ClickHouseConfig(); + Assert.assertEquals(config.getProductVersion(), "unknown"); + Assert.assertEquals(config.getProductRevision(), "unknown"); + Assert.assertEquals(config.getClientOsInfo(), + System.getProperty("os.name") + "/" + System.getProperty("os.version")); + Assert.assertEquals(config.getClientJvmInfo(), + System.getProperty("java.vm.name") + "/" + System.getProperty("java.vendor.version")); + Assert.assertEquals(config.getClientUser(), System.getProperty("user.name")); + Assert.assertEquals(config.getClientHost(), InetAddress.getLocalHost().getHostName()); + + Assert.assertEquals(ClickHouseClientOption.buildUserAgent(null, null), + "ClickHouse-JavaClient/unknown (" + System.getProperty("os.name") + "/" + + System.getProperty("os.version") + "; " + System.getProperty("java.vm.name") + "/" + + System.getProperty("java.vendor.version") + "; rv:unknown)"); + Assert.assertEquals(ClickHouseClientOption.buildUserAgent(null, null), + ClickHouseClientOption.buildUserAgent("", null)); + + config = new ClickHouseConfig( + Collections.singletonMap(ClickHouseClientOption.CLIENT_NAME, "custom client name")); + Assert.assertEquals(config.getClientName(), "custom client name"); + } } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java index e2ccbf4fd..07e9eaa34 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java @@ -20,6 +20,7 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseOption; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; import com.clickhouse.client.grpc.impl.ClickHouseGrpc; import com.clickhouse.client.logging.Logger; @@ -107,6 +108,8 @@ protected ClickHouseGrpcChannelFactory(ClickHouseConfig config, ClickHouseNode s protected abstract ManagedChannelBuilder getChannelBuilder(); + protected abstract String getDefaultUserAgent(); + @SuppressWarnings("unchecked") protected void setupRetry() { ManagedChannelBuilder builder = getChannelBuilder(); @@ -195,7 +198,15 @@ public ManagedChannel create() { setupSsl(); setupMisc(); - ManagedChannel c = getChannelBuilder().build(); + ManagedChannelBuilder builder = getChannelBuilder(); + String userAgent = config.getClientName(); + if (ClickHouseOption.DEFAULT_CLIENT_NAME.equals(userAgent)) { + userAgent = getDefaultUserAgent(); + } + String name = config.getProductName(); + builder.userAgent(ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name) ? userAgent + : new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString()); + ManagedChannel c = builder.build(); log.debug("Channel established: %s", c); return c; } diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java index 5431a0675..6e18b5b7b 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/NettyChannelFactoryImpl.java @@ -15,10 +15,13 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseSslMode; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; final class NettyChannelFactoryImpl extends ClickHouseGrpcChannelFactory { + private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "gRPC-Netty"); + private final NettyChannelBuilder builder; NettyChannelFactoryImpl(ClickHouseConfig config, ClickHouseNode server) { @@ -70,6 +73,11 @@ protected ManagedChannelBuilder getChannelBuilder() { return builder; } + @Override + protected String getDefaultUserAgent() { + return USER_AGENT; + } + @Override protected void setupSsl() { if (!config.isSsl()) { diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java index 01980ca82..645d43ed8 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/OkHttpChannelFactoryImpl.java @@ -9,9 +9,12 @@ import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseSslContextProvider; +import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.grpc.config.ClickHouseGrpcOption; final class OkHttpChannelFactoryImpl extends ClickHouseGrpcChannelFactory { + private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "gRPC-OkHttp"); + private final OkHttpChannelBuilder builder; OkHttpChannelFactoryImpl(ClickHouseConfig config, ClickHouseNode server) { @@ -30,6 +33,11 @@ protected ManagedChannelBuilder getChannelBuilder() { return builder; } + @Override + protected String getDefaultUserAgent() { + return USER_AGENT; + } + @Override protected void setupSsl() { if (!config.isSsl()) { diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java index 57ad19e3c..3867567c9 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java @@ -35,6 +35,7 @@ import org.apache.hc.core5.http.protocol.HttpContext; import org.apache.hc.core5.ssl.SSLContexts; import org.apache.hc.core5.util.Timeout; +import org.apache.hc.core5.util.VersionInfo; import java.io.BufferedReader; import java.io.ByteArrayInputStream; @@ -50,7 +51,6 @@ import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.TimeZone; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; @@ -78,8 +78,8 @@ private CloseableHttpClient newConnection() throws IOException { if (config.isSsl()) { r.register("https", SSLSocketFactory.create(config)); } - return HttpClientBuilder.create() - .setConnectionManager(new HttpConnectionManager(r.build(), config)).disableContentCompression().build(); + return HttpClientBuilder.create().setConnectionManager(new HttpConnectionManager(r.build(), config)) + .disableContentCompression().build(); } private ClickHouseHttpResponse buildResponse(CloseableHttpResponse response, ClickHouseConfig config, @@ -188,6 +188,11 @@ private void checkResponse(CloseableHttpResponse response) throws IOException { throw new IOException(errorMsg); } + @Override + protected final String getDefaultUserAgent() { + return HttpConnectionManager.USER_AGENT; + } + @Override protected boolean isReusable() { return true; @@ -264,9 +269,8 @@ static class SSLSocketFactory extends SSLConnectionSocketFactory { private final ClickHouseConfig config; private SSLSocketFactory(ClickHouseConfig config) throws SSLException { - super(Objects.requireNonNull( - ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, config) - .orElse(SSLContexts.createDefault())), + super(ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, config) + .orElse(SSLContexts.createDefault()), config.getSslMode() == ClickHouseSslMode.STRICT ? HttpsURLConnection.getDefaultHostnameVerifier() : (hostname, session) -> true); // NOSONAR @@ -284,6 +288,22 @@ public static SSLSocketFactory create(ClickHouseConfig config) throws SSLExcepti } static class HttpConnectionManager extends PoolingHttpClientConnectionManager { + private static final String PROVIDER = "Apache-HttpClient"; + private static final String USER_AGENT; + static { + String versionInfo = null; + try { + versionInfo = VersionInfo + .getSoftwareInfo(PROVIDER, "org.apache.hc.client5", HttpClientBuilder.class) + .split("\\s")[0]; + } catch (Throwable e) { // NOSONAR + // ignore + } + + USER_AGENT = ClickHouseClientOption.buildUserAgent(null, + versionInfo != null && !versionInfo.isEmpty() ? versionInfo : PROVIDER); + } + public HttpConnectionManager(Registry socketFactory, ClickHouseConfig config) { super(socketFactory); diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java index f9aaaf64a..3102a4a66 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java @@ -118,7 +118,8 @@ protected ClickHouseResponse send(ClickHouseRequest sealedRequest) throws Cli httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), sealedRequest.getExternalTables(), ClickHouseHttpConnection.buildUrl(server.getBaseUri(), sealedRequest), - ClickHouseHttpConnection.createDefaultHeaders(config, server), config, postAction); + ClickHouseHttpConnection.createDefaultHeaders(config, server, conn.getUserAgent()), config, + postAction); } else { httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), sealedRequest.getExternalTables(), null, null, config, postAction); diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java index edb531ffe..12a615c3c 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -185,7 +185,8 @@ static String buildUrl(String baseUrl, ClickHouseRequest request) { return builder.toString(); } - protected static Map createDefaultHeaders(ClickHouseConfig config, ClickHouseNode server) { + protected static Map createDefaultHeaders(ClickHouseConfig config, ClickHouseNode server, + String userAgent) { Map map = new LinkedHashMap<>(); boolean hasAuthorizationHeader = false; // add customer headers @@ -206,7 +207,7 @@ protected static Map createDefaultHeaders(ClickHouseConfig confi if (!config.getBoolOption(ClickHouseHttpOption.KEEP_ALIVE)) { map.put("connection", "Close"); } - map.put("user-agent", config.getClientName()); + map.put("user-agent", !ClickHouseChecker.isNullOrEmpty(userAgent) ? userAgent : config.getClientName()); ClickHouseCredentials credentials = server.getCredentials(config); if (credentials.useAccessToken()) { @@ -332,7 +333,7 @@ protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest r this.server = server; this.output = request.getOutputStream().orElse(null); this.url = buildUrl(server.getBaseUri(), request); - this.defaultHeaders = Collections.unmodifiableMap(createDefaultHeaders(config, server)); + this.defaultHeaders = Collections.unmodifiableMap(createDefaultHeaders(config, server, getUserAgent())); this.rm = request.getManager(); } @@ -358,6 +359,22 @@ protected String getBaseUrl() { return baseUrl; } + protected String getDefaultUserAgent() { + return config.getClientName(); + } + + protected final String getUserAgent() { + String name = config.getClientName(); + if (!ClickHouseOption.DEFAULT_CLIENT_NAME.equals(name)) { + return name; + } + + String userAgent = getDefaultUserAgent(); + name = config.getProductName(); + return ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name) ? userAgent + : new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString(); + } + /** * Creates a merged map. * diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java index 99ec2d425..47725cca3 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java @@ -42,6 +42,8 @@ public class HttpUrlConnectionImpl extends ClickHouseHttpConnection { private static final Logger log = LoggerFactory.getLogger(HttpUrlConnectionImpl.class); + private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "HttpURLConnection"); + private final HttpURLConnection conn; private ClickHouseHttpResponse buildResponse(Runnable postCloseAction) throws IOException { @@ -187,6 +189,11 @@ protected HttpUrlConnectionImpl(ClickHouseNode server, ClickHouseRequest requ conn = newConnection(url, true); } + @Override + protected final String getDefaultUserAgent() { + return USER_AGENT; + } + @Override protected boolean isReusable() { return false; diff --git a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java index eb084acad..dd7877b92 100644 --- a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java +++ b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java @@ -73,6 +73,8 @@ public List select(URI uri) { private static final Logger log = LoggerFactory.getLogger(HttpClientConnectionImpl.class); + private static final String USER_AGENT = ClickHouseClientOption.buildUserAgent(null, "HttpClient"); + private final HttpClient httpClient; private final HttpRequest pingRequest; @@ -248,6 +250,11 @@ private ClickHouseHttpResponse postString(ClickHouseConfig config, HttpRequest.B return buildResponse(config, r, postAction); } + @Override + protected final String getDefaultUserAgent() { + return USER_AGENT; + } + @Override protected ClickHouseHttpResponse post(String sql, ClickHouseInputStream data, List tables, String url, Map headers, ClickHouseConfig config, Runnable postAction) throws IOException { diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java index 08edf014d..6fc30f5ea 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java @@ -190,6 +190,66 @@ public void testAuthentication() throws ClickHouseException { } } + @Test(groups = "integration") + public void testUserAgent() throws Exception { + final ClickHouseNode server = getServer(); + final String sql = "select :uuid(String)"; + + String uuid = UUID.randomUUID().toString(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .query(ClickHouseParameterizedQuery.of(client.getConfig(), sql)) + .params(ClickHouseStringValue.of(uuid)) + .executeAndWait()) { + Assert.assertEquals(response.firstRecord().getValue(0).asString(), uuid); + } + ClickHouseClient.send(server, "SYSTEM FLUSH LOGS").get(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .query("select http_user_agent from system.query_log where query='select ''" + uuid + "'''") + .executeAndWait()) { + String result = response.firstRecord().getValue(0).asString(); + Assert.assertTrue(result.startsWith(client.getConfig().getProductName())); + Assert.assertTrue(result.indexOf("Http") > 0); + } + + uuid = UUID.randomUUID().toString(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .option(ClickHouseClientOption.PRODUCT_NAME, "MyCustomClient") + .query(ClickHouseParameterizedQuery.of(client.getConfig(), sql)) + .params(ClickHouseStringValue.of(uuid)) + .executeAndWait()) { + Assert.assertEquals(response.firstRecord().getValue(0).asString(), uuid); + } + ClickHouseClient.send(server, "SYSTEM FLUSH LOGS").get(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .query("select http_user_agent from system.query_log where query='select ''" + uuid + "'''") + .executeAndWait()) { + String result = response.firstRecord().getValue(0).asString(); + Assert.assertTrue(result.startsWith("MyCustomClient")); + Assert.assertTrue(result.indexOf("Http") > 0); + } + + uuid = UUID.randomUUID().toString(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .option(ClickHouseClientOption.CLIENT_NAME, "MyCustomClient") + .query(ClickHouseParameterizedQuery.of(client.getConfig(), sql)) + .params(ClickHouseStringValue.of(uuid)) + .executeAndWait()) { + Assert.assertEquals(response.firstRecord().getValue(0).asString(), uuid); + } + ClickHouseClient.send(server, "SYSTEM FLUSH LOGS").get(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = newRequest(client, server) + .query("select http_user_agent from system.query_log where query='select ''" + uuid + "'''") + .executeAndWait()) { + Assert.assertEquals(response.firstRecord().getValue(0).asString(), "MyCustomClient"); + } + } + @Override @Test(groups = "integration") public void testSession() throws ClickHouseException { diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java index 18a47d381..6675008f7 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpConnectionTest.java @@ -25,8 +25,8 @@ protected SimpleHttpConnection(ClickHouseNode server, ClickHouseRequest reque @Override protected ClickHouseHttpResponse post(String query, ClickHouseInputStream data, - List tables, String url, Map headers, ClickHouseConfig config, - Runnable postAction) throws IOException { + List tables, String url, Map headers, + ClickHouseConfig config, Runnable postAction) throws IOException { return null; } diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java index d58440b09..272efa0e0 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDriver.java @@ -28,7 +28,7 @@ /** * JDBC driver for ClickHouse. It takes a connection string like below for * connecting to ClickHouse server: - * {@code jdbc:clickhouse://[:@][:][/][?parameter1=value1¶meter2=value2]} + * {@code jdbc:(ch|clickhouse)[:]://[[:]@][:][/][?,[[,]]} * *

* For examples: diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java index 897248649..2c2c96039 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParser.java @@ -84,6 +84,7 @@ static Properties newProperties() { Properties props = new Properties(); props.setProperty(ClickHouseClientOption.ASYNC.getKey(), Boolean.FALSE.toString()); props.setProperty(ClickHouseClientOption.FORMAT.getKey(), ClickHouseFormat.RowBinaryWithNamesAndTypes.name()); + props.setProperty(ClickHouseClientOption.PRODUCT_NAME.getKey(), "ClickHouse-JdbcDriver"); return props; } diff --git a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java index 9fb68968e..b48040b5a 100644 --- a/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java +++ b/clickhouse-r2dbc/src/main/java/com/clickhouse/r2dbc/connection/ClickHouseConnection.java @@ -29,6 +29,8 @@ public class ClickHouseConnection implements Connection { private static final Logger log = LoggerFactory.getLogger(ClickHouseConnection.class); + private static final String PRODUCT_NAME = "ClickHouse-R2dbcDriver"; + public static final int DEFAULT_TIMEOUT_FOR_CONNECTION_HEALTH_CHECK = (Integer) ClickHouseClientOption.CONNECTION_TIMEOUT.getDefaultValue(); final ClickHouseClient client; final ClickHouseNode node; @@ -85,7 +87,7 @@ public Publisher commitTransaction() { */ @Override public Batch createBatch() { - ClickHouseRequest req = client.connect(node); + ClickHouseRequest req = client.connect(node).option(ClickHouseClientOption.PRODUCT_NAME, PRODUCT_NAME); if (isHttp()) { req = req.set("send_progress_in_http_headers", 1); } @@ -105,7 +107,7 @@ public Publisher createSavepoint(String s) { @Override public Statement createStatement(String sql) { - ClickHouseRequest req = client.connect(node); + ClickHouseRequest req = client.connect(node).option(ClickHouseClientOption.PRODUCT_NAME, PRODUCT_NAME); if (isHttp()) { req = req.set("send_progress_in_http_headers", 1); } From 21dc42b7cc67f361c93a8258102c3586db529948 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 19:44:37 +0800 Subject: [PATCH 2/8] Fix Apache HttpClient hostname verification issue --- .../com/clickhouse/client/http/ApacheHttpConnectionImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java index 3867567c9..a57311030 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ApacheHttpConnectionImpl.java @@ -25,6 +25,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.client5.http.socket.ConnectionSocketFactory; import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; import org.apache.hc.core5.http.Header; import org.apache.hc.core5.http.HttpRequest; @@ -54,7 +55,6 @@ import java.util.TimeZone; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLException; @@ -272,7 +272,7 @@ private SSLSocketFactory(ClickHouseConfig config) throws SSLException { super(ClickHouseSslContextProvider.getProvider().getSslContext(SSLContext.class, config) .orElse(SSLContexts.createDefault()), config.getSslMode() == ClickHouseSslMode.STRICT - ? HttpsURLConnection.getDefaultHostnameVerifier() + ? new DefaultHostnameVerifier() : (hostname, session) -> true); // NOSONAR this.config = config; } From de2b501196e2845b3039351c532371d26b04778f Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 19:45:09 +0800 Subject: [PATCH 3/8] Do NOT trigger build for doc update and shorten build info --- .github/workflows/build.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5e85b17ef..5b5aa8955 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,8 +7,10 @@ on: - master paths-ignore: - "**.md" - - "docs/**" + - "**/docs/**" - "**/CHANGELOG" + - "**/LICENSE" + - "**/NOTICE" pull_request: types: @@ -17,8 +19,10 @@ on: - reopened paths-ignore: - "**.md" - - "docs/**" + - "**/docs/**" - "**/CHANGELOG" + - "**/LICENSE" + - "**/NOTICE" workflow_dispatch: inputs: @@ -58,7 +62,7 @@ jobs: runs-on: ubuntu-latest needs: compile timeout-minutes: 10 - name: CLI client against ClickHouse 22.8 + name: CLI client + CH 22.8 steps: - name: Check out repository uses: actions/checkout@v3 @@ -102,7 +106,7 @@ jobs: clickhouse: ["21.8", "22.3", "22.8", "latest"] fail-fast: false timeout-minutes: 15 - name: Java client against ClickHouse ${{ matrix.clickhouse }} + name: Java client + CH ${{ matrix.clickhouse }} steps: - name: Check out repository uses: actions/checkout@v3 @@ -162,7 +166,7 @@ jobs: protocol: grpc fail-fast: false timeout-minutes: 15 - name: JDBC driver against ClickHouse ${{ matrix.clickhouse }} (${{ matrix.protocol }}) + name: JDBC driver + CH ${{ matrix.clickhouse }} (${{ matrix.protocol }}) steps: - name: Check out repository uses: actions/checkout@v3 @@ -222,7 +226,7 @@ jobs: r2dbc: ["1.0.0.RELEASE", "0.9.1.RELEASE"] fail-fast: false timeout-minutes: 10 - name: R2DBC ${{ matrix.r2dbc }} against ClickHouse ${{ matrix.clickhouse }} (${{ matrix.protocol }}) + name: R2DBC ${{ matrix.r2dbc }} + CH ${{ matrix.clickhouse }} (${{ matrix.protocol }}) steps: - name: Check out repository uses: actions/checkout@v3 From b97d4d34c59b21cdd92ec4ec999dc8a805e94e07 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 19:50:56 +0800 Subject: [PATCH 4/8] Fix CI build failure --- .../test/java/com/clickhouse/client/ClickHouseConfigTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java index 1f994457f..1e5792fef 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java @@ -85,7 +85,9 @@ public void testClientInfo() throws UnknownHostException { Assert.assertEquals(ClickHouseClientOption.buildUserAgent(null, null), "ClickHouse-JavaClient/unknown (" + System.getProperty("os.name") + "/" + System.getProperty("os.version") + "; " + System.getProperty("java.vm.name") + "/" - + System.getProperty("java.vendor.version") + "; rv:unknown)"); + + System.getProperty("java.vendor.version", + System.getProperty("java.vm.version", System.getProperty("java.version", "unknown"))) + + "; rv:unknown)"); Assert.assertEquals(ClickHouseClientOption.buildUserAgent(null, null), ClickHouseClientOption.buildUserAgent("", null)); From c63ead2a1bbe0479ee72e9ed32b929dfc09b46a2 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 19:56:35 +0800 Subject: [PATCH 5/8] Fix CI build failure --- .../test/java/com/clickhouse/client/ClickHouseConfigTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java index 1e5792fef..8f05839cf 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseConfigTest.java @@ -78,7 +78,8 @@ public void testClientInfo() throws UnknownHostException { Assert.assertEquals(config.getClientOsInfo(), System.getProperty("os.name") + "/" + System.getProperty("os.version")); Assert.assertEquals(config.getClientJvmInfo(), - System.getProperty("java.vm.name") + "/" + System.getProperty("java.vendor.version")); + System.getProperty("java.vm.name") + "/" + System.getProperty("java.vendor.version", + System.getProperty("java.vm.version", System.getProperty("java.version", "unknown")))); Assert.assertEquals(config.getClientUser(), System.getProperty("user.name")); Assert.assertEquals(config.getClientHost(), InetAddress.getLocalHost().getHostName()); From 09ad0d0af4a2f8ea57f34b71ad015de5689e77df Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 20:31:00 +0800 Subject: [PATCH 6/8] Fix JDBC test failures when protocol is gRPC --- .../client/grpc/ClickHouseGrpcChannelFactory.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java index 07e9eaa34..fe346f480 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java @@ -202,11 +202,13 @@ public ManagedChannel create() { String userAgent = config.getClientName(); if (ClickHouseOption.DEFAULT_CLIENT_NAME.equals(userAgent)) { userAgent = getDefaultUserAgent(); + + String name = config.getProductName(); + if (!ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name)) { + userAgent = new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString(); + } } - String name = config.getProductName(); - builder.userAgent(ClickHouseOption.DEFAULT_PRODUCT_NAME.equals(name) ? userAgent - : new StringBuilder(name).append(userAgent.substring(userAgent.indexOf('/'))).toString()); - ManagedChannel c = builder.build(); + ManagedChannel c = builder.userAgent(userAgent).build(); log.debug("Channel established: %s", c); return c; } From 61713be9eed76304c34dd2c8e5845c1c6a99c3de Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 20:43:39 +0800 Subject: [PATCH 7/8] Show detailed error message for flaky tests --- .../java/com/clickhouse/client/ClientIntegrationTest.java | 6 +++--- .../java/com/clickhouse/jdbc/ClickHouseConnectionTest.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java index 249ec1393..ec9cc1737 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -436,7 +436,7 @@ public void testCompression(ClickHouseFormat format, ClickHouseBufferingMode buf } catch (ClickHouseException e) { exp = e; } - Assert.assertEquals(exp.getErrorCode(), 81); + Assert.assertEquals(exp.getErrorCode(), 81, "Expected error code 81 but we got: " + exp.getMessage()); } } @@ -486,7 +486,7 @@ public void testNonExistDb() throws ClickHouseException { Assert.fail("Exception is excepted"); } catch (ExecutionException e) { ClickHouseException ce = ClickHouseException.of(e.getCause(), server); - Assert.assertEquals(ce.getErrorCode(), 81); + Assert.assertEquals(ce.getErrorCode(), 81, "Expected error code 81 but we got: " + ce.getMessage()); } catch (InterruptedException e) { Assert.fail("Failed execute due to interruption", e); } @@ -497,7 +497,7 @@ public void testNonExistDb() throws ClickHouseException { Assert.fail("Exception is excepted"); } catch (ExecutionException e) { ClickHouseException ce = ClickHouseException.of(e.getCause(), server); - Assert.assertEquals(ce.getErrorCode(), 81); + Assert.assertEquals(ce.getErrorCode(), 81, "Expected error code 81 but we got: " + ce.getMessage()); } catch (InterruptedException e) { Assert.fail("Failed execute due to interruption", e); } diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java index c5d334dbe..beaed7915 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java @@ -108,7 +108,7 @@ public void testNonExistDatabase() throws SQLException { exp = e; } Assert.assertNotNull(exp, "Should have SQLException since the database does not exist"); - Assert.assertEquals(exp.getErrorCode(), 81); + Assert.assertEquals(exp.getErrorCode(), 81, "Expected error code 81 but we got: " + exp.getMessage()); props.setProperty(JdbcConfig.PROP_CREATE_DATABASE, Boolean.TRUE.toString()); try (ClickHouseConnection conn = newConnection(props)) { From b46bd67e10f97d125bf654e1154f32ec06ab8fc7 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Sun, 8 Jan 2023 21:41:05 +0800 Subject: [PATCH 8/8] Add workaround for the flaky tests --- .../clickhouse/client/ClickHouseUtils.java | 20 ++++++++++++++++++- .../client/ClickHouseUtilsTest.java | 14 +++++++++++++ .../client/http/ClickHouseHttpConnection.java | 6 +++++- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java index 7ae260bbc..2495ae0bb 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java @@ -162,6 +162,24 @@ private static T findFirstService(Class serviceInterface) { return service; } + public static String toJavaByteArrayExpression(byte[] bytes) { + if (bytes == null) { + return "null"; + } + int len = bytes.length; + if (len == 0) { + return "{}"; + } + + String prefix = "(byte)0x"; + StringBuilder builder = new StringBuilder(10 * len).append('{'); + for (int i = 0; i < len; i++) { + builder.append(prefix).append(String.format("%02X", 0xFF & bytes[i])).append(','); + } + builder.setCharAt(builder.length() - 1, '}'); + return builder.toString(); + } + public static int indexOf(byte[] bytes, byte[] search) { if (bytes == null || search == null) { return -1; @@ -172,7 +190,7 @@ public static int indexOf(byte[] bytes, byte[] search) { } int blen = bytes.length; - outer: for (int i = 0, len = blen - slen + 1; i < len; i++) { + outer: for (int i = 0, len = blen - slen + 1; i < len; i++) { // NOSONAR for (int j = 0; j < slen; j++) { if (bytes[i + j] != search[j]) { continue outer; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java index 403a662bd..e395795cf 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java @@ -514,4 +514,18 @@ public void testParseJson() { Assert.assertEquals(ClickHouseUtils.parseJson("[null, 1, [2,3], {\"a\" : 1, \"c\": null, \"b\":2.2}]"), new Object[] { null, 1, new Object[] { 2, 3 }, map }); } + + @Test(groups = { "unit" }) + public void testToJavaByteArrayExpression() { + Assert.assertEquals(ClickHouseUtils.toJavaByteArrayExpression(null), "null"); + Assert.assertEquals(ClickHouseUtils.toJavaByteArrayExpression(new byte[0]), "{}"); + Assert.assertEquals(ClickHouseUtils.toJavaByteArrayExpression(new byte[3]), + "{(byte)0x00,(byte)0x00,(byte)0x00}"); + byte[] bytes = new byte[256]; + for (int i = 0; i < bytes.length; i++) { + bytes[i] = (byte) i; + } + Assert.assertEquals(ClickHouseUtils.toJavaByteArrayExpression(bytes), + "{(byte)0x00,(byte)0x01,(byte)0x02,(byte)0x03,(byte)0x04,(byte)0x05,(byte)0x06,(byte)0x07,(byte)0x08,(byte)0x09,(byte)0x0A,(byte)0x0B,(byte)0x0C,(byte)0x0D,(byte)0x0E,(byte)0x0F,(byte)0x10,(byte)0x11,(byte)0x12,(byte)0x13,(byte)0x14,(byte)0x15,(byte)0x16,(byte)0x17,(byte)0x18,(byte)0x19,(byte)0x1A,(byte)0x1B,(byte)0x1C,(byte)0x1D,(byte)0x1E,(byte)0x1F,(byte)0x20,(byte)0x21,(byte)0x22,(byte)0x23,(byte)0x24,(byte)0x25,(byte)0x26,(byte)0x27,(byte)0x28,(byte)0x29,(byte)0x2A,(byte)0x2B,(byte)0x2C,(byte)0x2D,(byte)0x2E,(byte)0x2F,(byte)0x30,(byte)0x31,(byte)0x32,(byte)0x33,(byte)0x34,(byte)0x35,(byte)0x36,(byte)0x37,(byte)0x38,(byte)0x39,(byte)0x3A,(byte)0x3B,(byte)0x3C,(byte)0x3D,(byte)0x3E,(byte)0x3F,(byte)0x40,(byte)0x41,(byte)0x42,(byte)0x43,(byte)0x44,(byte)0x45,(byte)0x46,(byte)0x47,(byte)0x48,(byte)0x49,(byte)0x4A,(byte)0x4B,(byte)0x4C,(byte)0x4D,(byte)0x4E,(byte)0x4F,(byte)0x50,(byte)0x51,(byte)0x52,(byte)0x53,(byte)0x54,(byte)0x55,(byte)0x56,(byte)0x57,(byte)0x58,(byte)0x59,(byte)0x5A,(byte)0x5B,(byte)0x5C,(byte)0x5D,(byte)0x5E,(byte)0x5F,(byte)0x60,(byte)0x61,(byte)0x62,(byte)0x63,(byte)0x64,(byte)0x65,(byte)0x66,(byte)0x67,(byte)0x68,(byte)0x69,(byte)0x6A,(byte)0x6B,(byte)0x6C,(byte)0x6D,(byte)0x6E,(byte)0x6F,(byte)0x70,(byte)0x71,(byte)0x72,(byte)0x73,(byte)0x74,(byte)0x75,(byte)0x76,(byte)0x77,(byte)0x78,(byte)0x79,(byte)0x7A,(byte)0x7B,(byte)0x7C,(byte)0x7D,(byte)0x7E,(byte)0x7F,(byte)0x80,(byte)0x81,(byte)0x82,(byte)0x83,(byte)0x84,(byte)0x85,(byte)0x86,(byte)0x87,(byte)0x88,(byte)0x89,(byte)0x8A,(byte)0x8B,(byte)0x8C,(byte)0x8D,(byte)0x8E,(byte)0x8F,(byte)0x90,(byte)0x91,(byte)0x92,(byte)0x93,(byte)0x94,(byte)0x95,(byte)0x96,(byte)0x97,(byte)0x98,(byte)0x99,(byte)0x9A,(byte)0x9B,(byte)0x9C,(byte)0x9D,(byte)0x9E,(byte)0x9F,(byte)0xA0,(byte)0xA1,(byte)0xA2,(byte)0xA3,(byte)0xA4,(byte)0xA5,(byte)0xA6,(byte)0xA7,(byte)0xA8,(byte)0xA9,(byte)0xAA,(byte)0xAB,(byte)0xAC,(byte)0xAD,(byte)0xAE,(byte)0xAF,(byte)0xB0,(byte)0xB1,(byte)0xB2,(byte)0xB3,(byte)0xB4,(byte)0xB5,(byte)0xB6,(byte)0xB7,(byte)0xB8,(byte)0xB9,(byte)0xBA,(byte)0xBB,(byte)0xBC,(byte)0xBD,(byte)0xBE,(byte)0xBF,(byte)0xC0,(byte)0xC1,(byte)0xC2,(byte)0xC3,(byte)0xC4,(byte)0xC5,(byte)0xC6,(byte)0xC7,(byte)0xC8,(byte)0xC9,(byte)0xCA,(byte)0xCB,(byte)0xCC,(byte)0xCD,(byte)0xCE,(byte)0xCF,(byte)0xD0,(byte)0xD1,(byte)0xD2,(byte)0xD3,(byte)0xD4,(byte)0xD5,(byte)0xD6,(byte)0xD7,(byte)0xD8,(byte)0xD9,(byte)0xDA,(byte)0xDB,(byte)0xDC,(byte)0xDD,(byte)0xDE,(byte)0xDF,(byte)0xE0,(byte)0xE1,(byte)0xE2,(byte)0xE3,(byte)0xE4,(byte)0xE5,(byte)0xE6,(byte)0xE7,(byte)0xE8,(byte)0xE9,(byte)0xEA,(byte)0xEB,(byte)0xEC,(byte)0xED,(byte)0xEE,(byte)0xEF,(byte)0xF0,(byte)0xF1,(byte)0xF2,(byte)0xF3,(byte)0xF4,(byte)0xF5,(byte)0xF6,(byte)0xF7,(byte)0xF8,(byte)0xF9,(byte)0xFA,(byte)0xFB,(byte)0xFC,(byte)0xFD,(byte)0xFE,(byte)0xFF}"); + } } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java index 12a615c3c..65ba3efe9 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -43,7 +43,7 @@ public abstract class ClickHouseHttpConnection implements AutoCloseable { private static final byte[] HEADER_BINARY_ENCODING = "content-transfer-encoding: binary\r\n\r\n" .getBytes(StandardCharsets.US_ASCII); - private static final byte[] ERROR_MSG_PREFIX = "Code: ".getBytes(StandardCharsets.US_ASCII); + private static final byte[] ERROR_MSG_PREFIX = "ode: ".getBytes(StandardCharsets.US_ASCII); private static final byte[] DOUBLE_DASH = new byte[] { '-', '-' }; private static final byte[] END_OF_NAME = new byte[] { '"', '\r', '\n' }; @@ -245,11 +245,15 @@ protected static String parseErrorFromException(String errorCode, String serverN int index = ClickHouseUtils.indexOf(bytes, ERROR_MSG_PREFIX); final String errorMsg; if (index > 0) { + bytes[--index] = (byte) 'C'; errorMsg = new String(bytes, index, bytes.length - index, StandardCharsets.UTF_8); } else if (!ClickHouseChecker.isNullOrBlank(errorCode)) { errorMsg = new StringBuilder().append("Code: ").append(errorCode).append(", server: ").append(serverName) .append(", ").append(new String(bytes, StandardCharsets.UTF_8)).toString(); } else { + // uncomment to debug + // log.debug("Unparsable error message[code=%s] returned from server[%s]: %s", + // errorCode, serverName, ClickHouseUtils.toJavaByteArrayExpression(bytes)); errorMsg = new String(bytes, StandardCharsets.UTF_8); } return errorMsg;