From 8d6ce581576f2219f4a06b903ca25449ed894dd6 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Mon, 11 Jul 2022 21:49:04 +0800 Subject: [PATCH 1/3] Clean up pom files --- clickhouse-benchmark/pom.xml | 3 +- clickhouse-cli-client/pom.xml | 65 ++++++----- clickhouse-client/pom.xml | 3 +- clickhouse-grpc-client/pom.xml | 47 ++++++-- clickhouse-http-client/pom.xml | 32 ++++-- clickhouse-jdbc/pom.xml | 102 ++++++------------ pom.xml | 30 ++++++ third-party-libraries/io.grpc/pom.xml | 14 ++- .../org.roaringbitmap/pom.xml | 14 ++- 9 files changed, 193 insertions(+), 117 deletions(-) diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index 76425ccfb..0025ed61f 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -8,10 +8,9 @@ clickhouse-benchmark - ${revision} jar - ${project.artifactId} + ClickHouse Client Benchmarks Benchmarks for ClickHouse clients https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-benchmark diff --git a/clickhouse-cli-client/pom.xml b/clickhouse-cli-client/pom.xml index 497d9ac5d..68ccff155 100644 --- a/clickhouse-cli-client/pom.xml +++ b/clickhouse-cli-client/pom.xml @@ -8,28 +8,21 @@ clickhouse-cli-client - ${revision} jar - ${project.artifactId} + ClickHouse Native Command-line Wrapper Wrapper of ClickHouse native command-line client https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-cli-client - - ${project.parent.groupId}.client.internal - - ${project.parent.groupId} clickhouse-client ${revision} - - - * - * - - + + + org.lz4 + lz4-java @@ -65,6 +58,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-failsafe-plugin + + + clickhouse-cli-client + + + org.apache.maven.plugins maven-shade-plugin @@ -77,13 +83,24 @@ true + shaded true true - shaded + + + net.jpountz + ${shade.base}.jpountz + + + + + com.clickhouse.client.cli + + @@ -104,17 +121,17 @@ - org.apache.maven.plugins - maven-compiler-plugin - - - org.apache.maven.plugins - maven-failsafe-plugin - - - clickhouse-cli-client - - + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + diff --git a/clickhouse-client/pom.xml b/clickhouse-client/pom.xml index 05e72f6c4..f41e148d8 100644 --- a/clickhouse-client/pom.xml +++ b/clickhouse-client/pom.xml @@ -8,10 +8,9 @@ clickhouse-client - ${revision} jar - ${project.artifactId} + ClickHouse Java Client Unified Java client for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-client diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index 0f8e89b4d..ea8c39b80 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -8,17 +8,12 @@ clickhouse-grpc-client - ${revision} jar - ${project.artifactId} + ClickHouse gRPC Client gRPC client for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-grpc-client - - ${project.parent.groupId}.client.internal - - ${project.parent.groupId} @@ -35,6 +30,10 @@ + + org.lz4 + lz4-java + com.google.code.gson @@ -88,6 +87,10 @@ + + org.apache.maven.plugins + maven-compiler-plugin + org.apache.maven.plugins maven-shade-plugin @@ -100,9 +103,9 @@ true + shaded true true - shaded com.google @@ -128,11 +131,20 @@ io.opencensus ${shade.base}.opencensus + + net.jpountz + ${shade.base}.jpountz + + + + com.clickhouse.client.grpc + + @@ -179,6 +191,10 @@ io.opencensus ${shade.base}.opencensus + + net.jpountz + ${shade.base}.jpountz + @@ -241,6 +257,10 @@ io.opencensus ${shade.base}.opencensus + + net.jpountz + ${shade.base}.jpountz + @@ -288,8 +308,17 @@ - org.apache.maven.plugins - maven-compiler-plugin + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + org.xolstice.maven.plugins diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index 2236a777e..184246f1c 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -8,22 +8,18 @@ clickhouse-http-client - ${revision} jar - ${project.artifactId} + ClickHouse HTTP Client HTTP client for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-http-client - - ${project.parent.groupId}.client.internal - - ${project.parent.groupId} clickhouse-client ${revision} + compile com.google.code.gson @@ -83,6 +79,10 @@ + + org.apache.maven.plugins + maven-compiler-plugin + org.apache.maven.plugins maven-shade-plugin @@ -95,9 +95,9 @@ true + shaded true true - shaded net.jpountz @@ -108,6 +108,11 @@ + + + com.clickhouse.client.http + + @@ -128,8 +133,17 @@ - org.apache.maven.plugins - maven-compiler-plugin + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index 2889d2edb..e43a3a3f9 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -8,28 +8,20 @@ clickhouse-jdbc - ${revision} jar - ${project.artifactId} + ClickHouse JDBC Driver JDBC driver for ClickHouse https://github.com/ClickHouse/clickhouse-jdbc/tree/master/clickhouse-jdbc 4.5.13 4.1.4 - com.clickhouse.jdbc.internal JDBC 4.2 - - ${project.parent.groupId} - clickhouse-client - ${revision} - provided - ${project.parent.groupId} clickhouse-cli-client @@ -45,7 +37,7 @@ ${project.parent.groupId} clickhouse-grpc-client - netty + shaded ${revision} @@ -92,13 +84,17 @@ httpmime ${httpclient.version} + - org.lz4 - lz4-java + ${project.parent.groupId} + clickhouse-client + ${revision} + provided - org.apache.commons - commons-compress + org.lz4 + lz4-java + provided @@ -181,25 +177,14 @@ - - io.github.git-commit-id - git-commit-id-maven-plugin - org.apache.maven.plugins maven-jar-plugin - - true - true - true - ${spec.title} ${spec.version} - ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -243,10 +228,6 @@ org.apache ${shade.base}.apache - - net.jpountz - ${shade.base}.jpountz - @@ -300,20 +281,6 @@ true true all - - - com.google.gson - ${shade.base}.gson - - - org.apache - ${shade.base}.apache - - - net.jpountz - ${shade.base}.jpountz - - @@ -334,7 +301,9 @@ *:* + com/google/** mozilla/** + org/** **/module-info.class META-INF/DEPENDENCIES META-INF/MANIFEST.MF @@ -357,16 +326,6 @@ true true grpc - - - com.google.gson - ${shade.base}.gson - - - net.jpountz - ${shade.base}.jpountz - - @@ -385,13 +344,13 @@ - ${project.parent.groupId}:clickhouse-http-client + ${project.parent.groupId}:clickhouse-cli-client ** - org.apache.httpcomponents:* + ${project.parent.groupId}:clickhouse-http-client ** @@ -399,7 +358,10 @@ *:* + com/google/** mozilla/** + org/** + ru/** **/darwin/** **/linux/** **/win32/** @@ -443,25 +405,13 @@ - ${project.parent.groupId}:clickhouse-grpc-client + ${project.parent.groupId}:clickhouse-cli-client ** - com.google.code.gson:* - - ** - - - - org.apache.commons:commons-compress - - ** - - - - org.apache.httpcomponents:* + ${project.parent.groupId}:clickhouse-grpc-client ** @@ -469,6 +419,7 @@ *:* + com/google/** mozilla/** org/** ru/** @@ -488,6 +439,19 @@ + + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index b7ddae4f3..651399512 100644 --- a/pom.xml +++ b/pom.xml @@ -103,6 +103,7 @@ 42.3.5 1.2.0 + ${project.groupId}.client.internal 3.3.0 3.10.1 @@ -407,6 +408,7 @@ ${flatten-plugin.version} true + all ossrh @@ -425,6 +427,7 @@ + true org.codehaus.mojo @@ -752,8 +755,17 @@ default-jar + + true + true + true + true + ${project.url} + ClickHouse, Inc. + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) ${git.commit.id.full} @@ -947,6 +959,14 @@ + + org.codehaus.mojo + flatten-maven-plugin + + + io.github.git-commit-id + git-commit-id-maven-plugin + org.apache.maven.plugins maven-jar-plugin @@ -955,8 +975,18 @@ default-jar + + true + true + true + true + ${project.url} + ClickHouse, Inc. + ${project.groupId} + ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + ${git.commit.id.full} diff --git a/third-party-libraries/io.grpc/pom.xml b/third-party-libraries/io.grpc/pom.xml index 52aff1b40..480697db5 100644 --- a/third-party-libraries/io.grpc/pom.xml +++ b/third-party-libraries/io.grpc/pom.xml @@ -9,7 +9,6 @@ io.grpc - ${revision} jar ${project.artifactId} @@ -112,6 +111,19 @@ + + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + + org.moditect moditect-maven-plugin diff --git a/third-party-libraries/org.roaringbitmap/pom.xml b/third-party-libraries/org.roaringbitmap/pom.xml index f85521e67..d13325aac 100644 --- a/third-party-libraries/org.roaringbitmap/pom.xml +++ b/third-party-libraries/org.roaringbitmap/pom.xml @@ -8,7 +8,6 @@ org.roaringbitmap - ${revision} jar ${project.artifactId} @@ -64,6 +63,19 @@ + + org.codehaus.mojo + flatten-maven-plugin + + + flatten + package + + flatten + + + + org.moditect moditect-maven-plugin From 8b8415fd19ad016123cbecf2422e5a1b0b49a6cd Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 13 Jul 2022 19:02:34 +0800 Subject: [PATCH 2/3] Fix build failure --- .../client/ClickHouseInputStream.java | 14 ++-- clickhouse-grpc-client/pom.xml | 29 +++++-- .../client/http/ClickHouseHttpClient.java | 17 +++- .../client/http/ClickHouseHttpConnection.java | 74 ++++++++--------- .../client/http/HttpUrlConnectionImpl.java | 2 +- .../client/http/HttpClientConnectionImpl.java | 39 ++++----- .../http/ClickHouseHttpConnectionTest.java | 3 +- clickhouse-jdbc/pom.xml | 81 ++++++++++++++----- .../jdbc/ClickHouseDatabaseMetaData.java | 2 +- .../jdbc/ClickHouseStatementTest.java | 6 +- 10 files changed, 173 insertions(+), 94 deletions(-) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java index bf3656248..2fec61da4 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseInputStream.java @@ -75,11 +75,13 @@ public abstract class ClickHouseInputStream extends InputStream { * @param compressionLevel compression level * @return non-null wrapped input stream */ - static ClickHouseInputStream wrap(ClickHouseFile file, InputStream input, int bufferSize, Runnable postCloseAction, - ClickHouseCompression compression, int compressionLevel) { + public static ClickHouseInputStream wrap(ClickHouseFile file, InputStream input, int bufferSize, + Runnable postCloseAction, ClickHouseCompression compression, int compressionLevel) { final ClickHouseInputStream chInput; if (compression == null || compression == ClickHouseCompression.NONE) { - chInput = new WrappedInputStream(file, input, bufferSize, postCloseAction); + chInput = input != EmptyInputStream.INSTANCE && input instanceof ClickHouseInputStream + ? (ClickHouseInputStream) input + : new WrappedInputStream(file, input, bufferSize, postCloseAction); } else { switch (compression) { case GZIP: @@ -485,7 +487,7 @@ public static File save(File file, InputStream in, int bufferSize, int timeout, tmp = File.createTempFile("chc", "data"); tmp.deleteOnExit(); } catch (IOException e) { - throw new IllegalStateException("Failed to create temp file", e); + throw new UncheckedIOException("Failed to create temp file", e); } } CompletableFuture data = CompletableFuture.supplyAsync(() -> { @@ -509,7 +511,9 @@ public static File save(File file, InputStream in, int bufferSize, int timeout, } catch (ExecutionException e) { Throwable cause = e.getCause(); if (cause instanceof UncheckedIOException) { - cause = ((UncheckedIOException) cause).getCause(); + throw ((UncheckedIOException) cause); + } else if (cause instanceof IOException) { + throw new UncheckedIOException((IOException) cause); } throw new IllegalStateException(cause); } diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index ea8c39b80..66279ffb2 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -39,15 +39,15 @@ com.google.code.gson gson + + org.apache.commons + commons-compress + io.grpc grpc-protobuf provided - - org.apache.commons - commons-compress - org.apache.tomcat @@ -119,6 +119,10 @@ okio ${shade.base}.okio + + org.apache + ${shade.base}.apache + io.grpc ${shade.base}.grpc @@ -153,7 +157,8 @@ android/** google/** javax/** - org/** + org/checkerframework/** + org/codehaus/** **/module-info.class META-INF/MANIFEST.MF META-INF/maven/** @@ -179,6 +184,10 @@ com.google ${shade.base}.google + + org.apache + ${shade.base}.apache + io.grpc ${shade.base}.grpc @@ -211,7 +220,8 @@ io/grpc/okhttp/** javax/** okio/** - org/** + org/checkerframework/** + org/codehaus/** **/module-info.class META-INF/MANIFEST.MF META-INF/maven/** @@ -245,6 +255,10 @@ okio ${shade.base}.okio + + org.apache + ${shade.base}.apache + io.grpc ${shade.base}.grpc @@ -275,7 +289,8 @@ google/** io/grpc/netty/** javax/** - org/** + org/checkerframework/** + org/codehaus/** **/module-info.class META-INF/MANIFEST.MF META-INF/maven/** diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java index 36acebe99..1fd8d4b07 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpClient.java @@ -9,6 +9,7 @@ import java.util.concurrent.CompletionException; import com.clickhouse.client.AbstractClient; +import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; @@ -29,7 +30,7 @@ public class ClickHouseHttpClient extends AbstractClient request) { // return false to suggest creating a new connection - return connection != null && connection.isReusable() && requestServer.equals(currentServer); + return connection != null && connection.isReusable() && requestServer.isSameEndpoint(currentServer); } @Override @@ -98,8 +99,18 @@ protected ClickHouseResponse send(ClickHouseRequest sealedRequest) throws Cli } log.debug("Query: %s", sql); - ClickHouseHttpResponse httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), - sealedRequest.getExternalTables(), null); + ClickHouseConfig config = sealedRequest.getConfig(); + final ClickHouseHttpResponse httpResponse; + if (conn.isReusable()) { + ClickHouseNode server = sealedRequest.getServer(); + httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), + sealedRequest.getExternalTables(), + ClickHouseHttpConnection.buildUrl(server.getBaseUri(), sealedRequest), + ClickHouseHttpConnection.createDefaultHeaders(config, server), config); + } else { + httpResponse = conn.post(sql, sealedRequest.getInputStream().orElse(null), + sealedRequest.getExternalTables(), null, null, config); + } return ClickHouseStreamResponse.of(httpResponse.getConfig(sealedRequest), httpResponse.getInputStream(), sealedRequest.getSettings(), null, httpResponse.summary); } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java index 2643692aa..64e73abe0 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -26,6 +26,11 @@ import com.clickhouse.client.http.config.ClickHouseHttpOption; public abstract class ClickHouseHttpConnection implements AutoCloseable { + private static StringBuilder appendQueryParameter(StringBuilder builder, String key, String value) { + return builder.append(urlEncode(key, StandardCharsets.UTF_8)).append('=') + .append(urlEncode(value, StandardCharsets.UTF_8)).append('&'); + } + static String urlEncode(String str, Charset charset) { if (charset == null) { charset = StandardCharsets.UTF_8; @@ -39,11 +44,6 @@ static String urlEncode(String str, Charset charset) { } } - private static StringBuilder appendQueryParameter(StringBuilder builder, String key, String value) { - return builder.append(urlEncode(key, StandardCharsets.UTF_8)).append('=') - .append(urlEncode(value, StandardCharsets.UTF_8)).append('&'); - } - static String buildQueryParams(ClickHouseRequest request) { if (request == null) { return ""; @@ -146,26 +146,7 @@ static String buildUrl(String baseUrl, ClickHouseRequest request) { return builder.toString(); } - protected final ClickHouseConfig config; - protected final ClickHouseNode server; - protected final Map defaultHeaders; - - protected final ClickHouseOutputStream output; - - protected final String url; - - protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest request) { - if (server == null || request == null) { - throw new IllegalArgumentException("Non-null server and request are required"); - } - - this.config = request.getConfig(); - this.server = server; - - this.output = request.getOutputStream().orElse(null); - - this.url = buildUrl(server.getBaseUri(), request); - + protected static Map createDefaultHeaders(ClickHouseConfig config, ClickHouseNode server) { Map map = new LinkedHashMap<>(); // add customer headers map.putAll(ClickHouseUtils.getKeyValuePairs((String) config.getOption(ClickHouseHttpOption.CUSTOM_HEADERS))); @@ -201,8 +182,25 @@ protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest r && config.getRequestCompressAlgorithm() != ClickHouseCompression.LZ4) { map.put("Content-Encoding", config.getRequestCompressAlgorithm().encoding()); } + return map; + } - this.defaultHeaders = Collections.unmodifiableMap(map); + protected final ClickHouseConfig config; + protected final ClickHouseNode server; + protected final ClickHouseOutputStream output; + protected final String url; + protected final Map defaultHeaders; + + protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest request) { + if (server == null || request == null) { + throw new IllegalArgumentException("Non-null server and request are required"); + } + + this.config = request.getConfig(); + this.server = server; + this.output = request.getOutputStream().orElse(null); + this.url = buildUrl(server.getBaseUri(), request); + this.defaultHeaders = Collections.unmodifiableMap(createDefaultHeaders(config, server)); } protected void closeQuietly() { @@ -231,11 +229,13 @@ protected String getBaseUrl() { * Creates a merged map. * * @param requestHeaders request headers - * @return + * @return non-null merged headers */ protected Map mergeHeaders(Map requestHeaders) { if (requestHeaders == null || requestHeaders.isEmpty()) { return defaultHeaders; + } else if (isReusable()) { + return requestHeaders; } Map merged = new LinkedHashMap<>(); @@ -256,13 +256,15 @@ protected Map mergeHeaders(Map requestHeaders) { * @param query non-blank query * @param data optionally input stream for batch updating * @param tables optionally external tables for query + * @param url optionally url * @param headers optionally request headers + * @param config optionally configuration * @return response * @throws IOException when error occured posting request and/or server failed * to respond */ protected abstract ClickHouseHttpResponse post(String query, InputStream data, List tables, - Map headers) throws IOException; + String url, Map headers, ClickHouseConfig config) throws IOException; /** * Checks whether the connection is reusable or not. This method will be called @@ -287,36 +289,36 @@ protected boolean isReusable() { public abstract boolean ping(int timeout); public ClickHouseHttpResponse update(String query) throws IOException { - return post(query, null, null, null); + return post(query, null, null, null, null, null); } public ClickHouseHttpResponse update(String query, Map headers) throws IOException { - return post(query, null, null, headers); + return post(query, null, null, null, headers, null); } public ClickHouseHttpResponse update(String query, InputStream data) throws IOException { - return post(query, data, null, null); + return post(query, data, null, null, null, null); } public ClickHouseHttpResponse update(String query, InputStream data, Map headers) throws IOException { - return post(query, data, null, headers); + return post(query, data, null, null, headers, null); } public ClickHouseHttpResponse query(String query) throws IOException { - return post(query, null, null, null); + return post(query, null, null, null, null, null); } public ClickHouseHttpResponse query(String query, Map headers) throws IOException { - return post(query, null, null, headers); + return post(query, null, null, null, headers, null); } public ClickHouseHttpResponse query(String query, List tables) throws IOException { - return post(query, null, tables, null); + return post(query, null, tables, null, null, null); } public ClickHouseHttpResponse query(String query, List tables, Map headers) throws IOException { - return post(query, null, tables, headers); + return post(query, null, tables, null, headers, null); } } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java index abd8729f7..4346e3726 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/HttpUrlConnectionImpl.java @@ -202,7 +202,7 @@ protected boolean isReusable() { @Override protected ClickHouseHttpResponse post(String sql, InputStream data, List tables, - Map headers) throws IOException { + String url, Map headers, ClickHouseConfig config) throws IOException { Charset charset = StandardCharsets.US_ASCII; byte[] boundary = null; if (tables != null && !tables.isEmpty()) { diff --git a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java index d46c77a40..1a0b5213e 100644 --- a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java +++ b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java @@ -2,6 +2,7 @@ import com.clickhouse.client.ClickHouseChecker; import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseConfig; import com.clickhouse.client.ClickHouseDataStreamFactory; import com.clickhouse.client.ClickHouseFormat; import com.clickhouse.client.ClickHouseInputStream; @@ -50,12 +51,11 @@ public class HttpClientConnectionImpl extends ClickHouseHttpConnection { private static final Logger log = LoggerFactory.getLogger(HttpClientConnectionImpl.class); - private static final int MAX_RETRIES = 1; - private final HttpClient httpClient; private final HttpRequest pingRequest; - private ClickHouseHttpResponse buildResponse(HttpResponse r) throws IOException { + private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpResponse r) + throws IOException { HttpHeaders headers = r.headers(); String displayName = headers.firstValue("X-ClickHouse-Server-Display-Name").orElse(server.getHost()); String queryId = headers.firstValue("X-ClickHouse-Query-Id").orElse(""); @@ -90,7 +90,10 @@ private ClickHouseHttpResponse buildResponse(HttpResponse r) throws source = checkResponse(r).body(); action = this::closeQuietly; } - return new ClickHouseHttpResponse(this, ClickHouseClient.getResponseInputStream(config, source, action), + + return new ClickHouseHttpResponse(this, + ClickHouseInputStream.wrap(null, source, config.getReadBufferSize(), action, + config.getResponseCompressAlgorithm(), config.getResponseCompressLevel()), displayName, queryId, summary, format, timeZone); } @@ -125,10 +128,8 @@ protected HttpClientConnectionImpl(ClickHouseNode server, ClickHouseRequest r throws IOException { super(server, request); - HttpClient.Builder builder = HttpClient.newBuilder() - .version(Version.HTTP_1_1) - .connectTimeout(Duration.ofMillis(config.getConnectionTimeout())) - .followRedirects(Redirect.NORMAL); + HttpClient.Builder builder = HttpClient.newBuilder().version(Version.HTTP_1_1) + .connectTimeout(Duration.ofMillis(config.getConnectionTimeout())).followRedirects(Redirect.NORMAL); if (executor != null) { builder.executor(executor); } @@ -153,8 +154,8 @@ private CompletableFuture> postRequest(HttpRequest req responseInfo -> new ClickHouseResponseHandler(config.getMaxQueuedBuffers(), config.getSocketTimeout())); } - private ClickHouseHttpResponse postStream(HttpRequest.Builder reqBuilder, String boundary, String sql, - InputStream data, List tables) throws IOException { + private ClickHouseHttpResponse postStream(ClickHouseConfig config, HttpRequest.Builder reqBuilder, String boundary, + String sql, InputStream data, List tables) throws IOException { ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance().createPipedOutputStream(config, null); reqBuilder.POST(HttpRequest.BodyPublishers.ofInputStream(stream::getInputStream)); @@ -217,10 +218,11 @@ private ClickHouseHttpResponse postStream(HttpRequest.Builder reqBuilder, String } } - return buildResponse(r); + return buildResponse(config, r); } - private ClickHouseHttpResponse postString(HttpRequest.Builder reqBuilder, String sql) throws IOException { + private ClickHouseHttpResponse postString(ClickHouseConfig config, HttpRequest.Builder reqBuilder, String sql) + throws IOException { reqBuilder.POST(HttpRequest.BodyPublishers.ofString(sql)); HttpResponse r; try { @@ -236,15 +238,16 @@ private ClickHouseHttpResponse postString(HttpRequest.Builder reqBuilder, String throw new IOException("Failed to post query", cause); } } - return buildResponse(r); + return buildResponse(config, r); } @Override protected ClickHouseHttpResponse post(String sql, InputStream data, List tables, - Map headers) throws IOException { + String url, Map headers, ClickHouseConfig config) throws IOException { + ClickHouseConfig c = config == null ? this.config : config; HttpRequest.Builder reqBuilder = HttpRequest.newBuilder() - .uri(URI.create(url)) - .timeout(Duration.ofMillis(config.getSocketTimeout())); + .uri(URI.create(ClickHouseChecker.isNullOrEmpty(url) ? this.url : url)) + .timeout(Duration.ofMillis(c.getSocketTimeout())); String boundary = null; if (tables != null && !tables.isEmpty()) { boundary = UUID.randomUUID().toString(); @@ -260,8 +263,8 @@ protected ClickHouseHttpResponse post(String sql, InputStream data, List reque @Override protected ClickHouseHttpResponse post(String query, InputStream data, List tables, - Map headers) throws IOException { + String url, Map headers, ClickHouseConfig config) throws IOException { return null; } diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index e43a3a3f9..edccc6bc5 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -235,13 +235,9 @@ - com.clickhouse.jdbc - + ${project.groupId}.jdbc ${spec.title} ${spec.version} - ${project.name} - ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -287,13 +283,9 @@ - com.clickhouse.jdbc - + ${project.groupId}.jdbc ${spec.title} ${spec.version} - ${project.name} - ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -332,13 +324,9 @@ - com.clickhouse.jdbc - + ${project.groupId}.jdbc ${spec.title} ${spec.version} - ${project.name} - ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -393,13 +381,9 @@ - com.clickhouse.jdbc - + ${project.groupId}.jdbc ${spec.title} ${spec.version} - ${project.name} - ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) @@ -437,6 +421,63 @@ + + shade-cli + package + + shade + + + true + true + true + cli + + + + + + + ${project.groupId}.jdbc + ${spec.title} + ${spec.version} + + + + + + ${project.parent.groupId}:clickhouse-grpc-client + + ** + + + + ${project.parent.groupId}:clickhouse-http-client + + ** + + + + *:* + + com/google/** + mozilla/** + org/** + ru/** + **/darwin/** + **/linux/** + **/win32/** + **/module-info.class + META-INF/DEPENDENCIES + META-INF/MANIFEST.MF + META-INF/maven/** + META-INF/native-image/** + META-INF/*.xml + + + + + diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java index a16d67c82..55eec3345 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/ClickHouseDatabaseMetaData.java @@ -824,7 +824,7 @@ public ResultSet getColumns(String catalog, String schemaPattern, String tableNa + "toInt32(position(type, 'Nullable(') >= 1 ? :defaultNullable : :defaultNonNull) as NULLABLE, :comment as REMARKS, default_expression as COLUMN_DEF, " + "0 as SQL_DATA_TYPE, 0 as SQL_DATETIME_SUB, cast(null as Nullable(Int32)) as CHAR_OCTET_LENGTH, position as ORDINAL_POSITION, " + "position(type, 'Nullable(') >= 1 ? 'YES' : 'NO' as IS_NULLABLE, null as SCOPE_CATALOG, null as SCOPE_SCHEMA, null as SCOPE_TABLE, " - + "null as SOURCE_DATA_TYPE, 'NO' as IS_AUTOINCREMENT, 'NO' as IS_GENERATEDCOLUMN from system.columns\n" + + "null as SOURCE_DATA_TYPE, 'NO' as IS_AUTOINCREMENT, 'NO' as IS_GENERATEDCOLUMN from system.columns " + "where database like :database and table like :table and name like :column", params); return query(sql, (i, r) -> { String typeName = r.getValue("TYPE_NAME").asString(); diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java index b79ac7d15..4861c29e8 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java @@ -1,5 +1,6 @@ package com.clickhouse.jdbc; +import java.io.IOException; import java.sql.Array; import java.sql.BatchUpdateException; import java.sql.Connection; @@ -81,8 +82,9 @@ public void testSocketTimeout() throws SQLException { stmt.executeQuery("select sleep(3)"); Assert.fail("Should throw timeout exception"); } catch (SQLException e) { - Assert.assertTrue(e.getCause() instanceof java.net.SocketTimeoutException, - "Should throw SocketTimeoutException"); + Assert.assertTrue(e.getCause() instanceof java.net.SocketTimeoutException + || e.getCause() instanceof IOException, + "Should throw SocketTimeoutException or HttpTimeoutException"); } } From 52de0296d5787d6c832e26461bd9891fb4dfca94 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Wed, 13 Jul 2022 19:48:43 +0800 Subject: [PATCH 3/3] Fix incorrect error code when using HttpClient on 21.3 --- .../client/http/HttpClientConnectionImpl.java | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java index 1a0b5213e..cb2a027b6 100644 --- a/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java +++ b/clickhouse-http-client/src/main/java11/com/clickhouse/client/http/HttpClientConnectionImpl.java @@ -10,6 +10,7 @@ import com.clickhouse.client.ClickHousePipedOutputStream; import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseSslContextProvider; +import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.http.config.ClickHouseHttpOption; import com.clickhouse.client.logging.Logger; @@ -17,6 +18,8 @@ import java.io.BufferedReader; import java.io.BufferedWriter; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; @@ -79,7 +82,7 @@ private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpRespon source = ClickHouseInputStream.empty(); action = () -> { try (OutputStream o = output) { - ClickHouseInputStream.pipe(checkResponse(r).body(), o, config.getWriteBufferSize()); + ClickHouseInputStream.pipe(checkResponse(config, r).body(), o, config.getWriteBufferSize()); } catch (IOException e) { throw new UncheckedIOException("Failed to redirect response to given output stream", e); } finally { @@ -87,7 +90,7 @@ private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpRespon } }; } else { - source = checkResponse(r).body(); + source = checkResponse(config, r).body(); action = this::closeQuietly; } @@ -97,23 +100,34 @@ private ClickHouseHttpResponse buildResponse(ClickHouseConfig config, HttpRespon displayName, queryId, summary, format, timeZone); } - private HttpResponse checkResponse(HttpResponse r) throws IOException { + private HttpResponse checkResponse(ClickHouseConfig config, HttpResponse r) + throws IOException { if (r.statusCode() != HttpURLConnection.HTTP_OK) { - // TODO get exception from response header, for example: - // X-ClickHouse-Exception-Code: 47 - StringBuilder builder = new StringBuilder(); - try (Reader reader = new InputStreamReader( - ClickHouseClient.getResponseInputStream(config, r.body(), this::closeQuietly), - StandardCharsets.UTF_8)) { - int c = 0; - while ((c = reader.read()) != -1) { - builder.append((char) c); + String errorCode = r.headers().firstValue("X-ClickHouse-Exception-Code").orElse(""); + // String encoding = r.headers().firstValue("Content-Encoding"); + String serverName = r.headers().firstValue("X-ClickHouse-Server-Display-Name").orElse(""); + + String errorMsg; + int bufferSize = (int) ClickHouseClientOption.BUFFER_SIZE.getDefaultValue(); + ByteArrayOutputStream output = new ByteArrayOutputStream(bufferSize); + ClickHouseInputStream.pipe(r.body(), output, bufferSize); + byte[] bytes = output.toByteArray(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + ClickHouseClient.getResponseInputStream(config, new ByteArrayInputStream(bytes), + this::closeQuietly), + StandardCharsets.UTF_8))) { + StringBuilder builder = new StringBuilder(); + while ((errorMsg = reader.readLine()) != null) { + builder.append(errorMsg).append('\n'); } + errorMsg = builder.toString(); } catch (IOException e) { - log.warn("Error while reading error message", e); + log.warn("Error while reading error message[code=%s] from server [%s]", errorCode, serverName, e); + errorMsg = new String(bytes, StandardCharsets.UTF_8); } - throw new IOException(builder.toString()); + throw new IOException(errorMsg); } return r;