From a1a617c50bafb3a03d99f33f3a7a4058635fa49b Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Thu, 23 Jun 2022 19:14:24 +0800 Subject: [PATCH 1/3] Change default sslmode to strict for security reason --- .../com/clickhouse/client/ClickHouseNode.java | 2 +- .../clickhouse/client/ClickHouseNodeTest.java | 16 ++++++++-------- .../clickhouse/client/ClickHouseNodesTest.java | 3 ++- .../internal/ClickHouseJdbcUrlParserTest.java | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java index 8045bdade..5a0bf54e5 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java @@ -801,7 +801,7 @@ public static ClickHouseNode of(URI uri, ClickHouseNode template) { } if (protocol != ClickHouseProtocol.POSTGRESQL && scheme.charAt(scheme.length() - 1) == 's') { params.put(ClickHouseClientOption.SSL.getKey(), Boolean.TRUE.toString()); - params.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.NONE.name()); + params.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); } ClickHouseCredentials credentials = template.credentials; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java index 223d9ab56..ac72a0b87 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodeTest.java @@ -169,7 +169,7 @@ public void testInvalidNodes() { public void testValidNodes() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.SSL.getKey(), "false"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); options.put(ClickHouseClientOption.DATABASE.getKey(), "db1"); Set tags = new HashSet<>(); @@ -183,7 +183,7 @@ public void testValidNodes() { public void testSecureNode() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.SSL.getKey(), "true"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); options.put(ClickHouseClientOption.DATABASE.getKey(), "db1"); Assert.assertEquals(ClickHouseNode.of("https://node1:443/db1"), @@ -218,7 +218,7 @@ public void testSingleWordNode() { public void testNodeWithProtocol() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.SSL.getKey(), "true"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); for (ClickHouseProtocol p : ClickHouseProtocol.values()) { Assert.assertEquals(ClickHouseNode.of(p.name() + ":///?#"), @@ -254,7 +254,7 @@ public void testNodeWithHostAndPort() { public void testNodeWithDatabase() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.SSL.getKey(), "true"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); Assert.assertEquals(ClickHouseNode.of("grpcs://node1:19100/"), new ClickHouseNode("node1", ClickHouseProtocol.GRPC, 19100, null, options, null)); @@ -324,13 +324,13 @@ public void testNodeWithOptions() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.ASYNC.getKey(), "false"); options.put(ClickHouseClientOption.SSL.getKey(), "true"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); options.put(ClickHouseClientOption.CONNECTION_TIMEOUT.getKey(), "500"); for (String uri : new String[] { "https://node1?!async&ssl&connect_timeout=500", - "http://node1?async=false&ssl=true&sslmode=NONE&connect_timeout=500", - "http://node1?&&&&async=false&ssl&&&&&sslmode=NONE&connect_timeout=500&&&", + "http://node1?async=false&ssl=true&sslmode=STRICT&connect_timeout=500", + "http://node1?&&&&async=false&ssl&&&&&sslmode=STRICT&connect_timeout=500&&&", }) { Assert.assertEquals(ClickHouseNode.of(uri), new ClickHouseNode("node1", ClickHouseProtocol.HTTP, @@ -379,7 +379,7 @@ public void testQueryWithSlash() throws Exception { Assert.assertEquals(server.toUri(), new URI("http://localhost:1234?/a/b/c=d")); Assert.assertEquals(ClickHouseNode.of("https://myserver/db/1/2/3?a%20=%201&b=/root/my.crt").toUri(), - new URI("http://myserver:8443/db/1/2/3?ssl=true&sslmode=NONE&a%20=%201&b=/root/my.crt")); + new URI("http://myserver:8443/db/1/2/3?ssl=true&sslmode=STRICT&a%20=%201&b=/root/my.crt")); } @Test(groups = { "integration" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java index 3e37b8735..7357adbac 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseNodesTest.java @@ -13,6 +13,7 @@ import com.clickhouse.client.ClickHouseNode.Status; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; +import com.clickhouse.client.config.ClickHouseSslMode; import org.testng.Assert; import org.testng.annotations.Test; @@ -213,7 +214,7 @@ public void testSingleNodeList() { Map options = new HashMap<>(); options.put(ClickHouseClientOption.SSL.getKey(), "true"); - options.put(ClickHouseClientOption.SSL_MODE.getKey(), "NONE"); + options.put(ClickHouseClientOption.SSL_MODE.getKey(), ClickHouseSslMode.STRICT.name()); options.put(ClickHouseClientOption.DATABASE.getKey(), "db1"); Assert.assertEquals(ClickHouseNodes.of("https://node1:443/db1").nodes.get(0), diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java index da192f347..f3af0cd63 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/internal/ClickHouseJdbcUrlParserTest.java @@ -69,12 +69,12 @@ public void testParseAbbrevation() throws SQLException, URISyntaxException { info = ClickHouseJdbcUrlParser.parse("jdbc:ch:https://:letmein@[::1]:3218/db1?user=aaa", null); Assert.assertEquals(info.getServer().toUri(ClickHouseJdbcUrlParser.JDBC_CLICKHOUSE_PREFIX), - new URI("jdbc:clickhouse:http://[::1]:3218/db1?ssl=true&sslmode=NONE")); + new URI("jdbc:clickhouse:http://[::1]:3218/db1?ssl=true&sslmode=STRICT")); Assert.assertEquals(info.getServer(), ClickHouseNode.builder().host("[::1]") .port(ClickHouseProtocol.HTTP, 3218) .database("db1") .credentials(ClickHouseCredentials.fromUserAndPassword("aaa", "letmein")) - .addOption("ssl", "true").addOption("sslmode", "NONE").build()); + .addOption("ssl", "true").addOption("sslmode", "STRICT").build()); Assert.assertEquals(info.getServer().getCredentials().orElse(null), ClickHouseCredentials.fromUserAndPassword("aaa", "letmein")); } @@ -98,13 +98,13 @@ public void testParse() throws SQLException, URISyntaxException { info = ClickHouseJdbcUrlParser.parse("jdbc:ch:https://:letmein@127.0.0.1:3218/db1", null); Assert.assertEquals(info.getServer().toUri(ClickHouseJdbcUrlParser.JDBC_CLICKHOUSE_PREFIX), - new URI("jdbc:clickhouse:http://127.0.0.1:3218/db1?ssl=true&sslmode=NONE")); + new URI("jdbc:clickhouse:http://127.0.0.1:3218/db1?ssl=true&sslmode=STRICT")); Assert.assertEquals(info.getServer(), ClickHouseNode.builder().host("127.0.0.1") .port(ClickHouseProtocol.HTTP, 3218).database("db1") .credentials(ClickHouseCredentials .fromUserAndPassword((String) ClickHouseDefaults.USER .getEffectiveDefaultValue(), "letmein")) - .addOption("ssl", "true").addOption("sslmode", "NONE") + .addOption("ssl", "true").addOption("sslmode", "STRICT") .build()); } From 3b48e07358aa23bc9def3ce9c3ded401e135a41b Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Thu, 23 Jun 2022 19:14:57 +0800 Subject: [PATCH 2/3] support custom writer for streaming --- .../clickhouse/client/ClickHouseRequest.java | 85 +++++++++++++++++++ .../client/ClientIntegrationTest.java | 41 +++++++++ 2 files changed, 126 insertions(+) diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java index 1bead6124..e188f9450 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java @@ -1,8 +1,10 @@ package com.clickhouse.client; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -14,8 +16,10 @@ import java.util.Map; import java.util.Objects; import java.util.Map.Entry; +import java.util.concurrent.CancellationException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicReference; import java.util.Optional; import java.util.Properties; @@ -39,6 +43,8 @@ public class ClickHouseRequest> implement * Mutation request. */ public static class Mutation extends ClickHouseRequest { + private ClickHouseWriter writer; + protected Mutation(ClickHouseRequest request, boolean sealed) { super(request.getClient(), request.server, request.serverRef, request.options, sealed); this.settings.putAll(request.settings); @@ -105,6 +111,20 @@ public Mutation format(ClickHouseFormat format) { return super.format(format); } + /** + * Sets custom writer for streaming. This will create a piped stream between the + * writer and ClickHouse server. + * + * @param writer writer + * @return mutation request + */ + public Mutation data(ClickHouseWriter writer) { + checkSealed(); + + this.writer = changeProperty(PROP_WRITER, this.writer, writer); + return this; + } + /** * Loads data from given file which may or may not be compressed. * @@ -197,6 +217,70 @@ public Mutation data(ClickHouseDeferredValue input) { return this; } + @Override + public CompletableFuture execute() { + if (writer != null) { + ClickHouseConfig c = getConfig(); + ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance() + .createPipedOutputStream(c, null); + data(stream.getInputStream()); + CompletableFuture future = null; + if (c.isAsync()) { + future = getClient().execute(isSealed() ? this : seal()); + } + try (ClickHouseOutputStream out = stream) { + writer.write(out); + } catch (IOException e) { + throw new CompletionException(e); + } + if (future != null) { + return future; + } + } + + return getClient().execute(isSealed() ? this : seal()); + } + + @Override + public ClickHouseResponse executeAndWait() throws ClickHouseException { + if (writer != null) { + ClickHouseConfig c = getConfig(); + ClickHousePipedOutputStream stream = ClickHouseDataStreamFactory.getInstance() + .createPipedOutputStream(c, null); + data(stream.getInputStream()); + CompletableFuture future = null; + if (c.isAsync()) { + future = getClient().execute(isSealed() ? this : seal()); + } + try (ClickHouseOutputStream out = stream) { + writer.write(out); + } catch (IOException e) { + throw ClickHouseException.of(e, getServer()); + } + if (future != null) { + try { + return future.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw ClickHouseException.forCancellation(e, getServer()); + } catch (CancellationException e) { + throw ClickHouseException.forCancellation(e, getServer()); + } catch (ExecutionException | UncheckedIOException e) { + Throwable cause = e.getCause(); + if (cause == null) { + cause = e; + } + throw cause instanceof ClickHouseException ? (ClickHouseException) cause + : ClickHouseException.of(cause, getServer()); + } catch (RuntimeException e) { // unexpected + throw ClickHouseException.of(e, getServer()); + } + } + } + + return getClient().executeAndWait(isSealed() ? this : seal()); + } + /** * Sends mutation requets for execution. Same as * {@code client.execute(request.seal())}. @@ -256,6 +340,7 @@ public Mutation seal() { static final String PROP_PREPARED_QUERY = "preparedQuery"; static final String PROP_QUERY = "query"; static final String PROP_QUERY_ID = "queryId"; + static final String PROP_WRITER = "writer"; private final boolean sealed; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java index 091fec959..b48460eef 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -21,6 +21,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import com.clickhouse.client.ClickHouseClientBuilder.Agent; @@ -909,6 +910,46 @@ public void testCustomRead() throws Exception { Assert.assertEquals(count, 1000L); } + @Test(groups = { "integration" }) + public void testCustomWriter() throws Exception { + ClickHouseNode server = getServer(); + ClickHouseClient.send(server, "drop table if exists test_custom_writer", + "create table test_custom_writer(a Int8) engine=Memory") + .get(); + + try (ClickHouseClient client = getClient()) { + AtomicInteger i = new AtomicInteger(1); + ClickHouseRequest.Mutation req = client.connect(server).write().format(ClickHouseFormat.RowBinary) + .table("test_custom_writer").data(o -> { + o.write(i.getAndIncrement()); + }); + for (boolean b : new boolean[] { true, false }) { + req.option(ClickHouseClientOption.ASYNC, b); + + try (ClickHouseResponse resp = req.send().get()) { + Assert.assertNotNull(resp); + } + + try (ClickHouseResponse resp = req.sendAndWait()) { + Assert.assertNotNull(resp); + } + + try (ClickHouseResponse resp = req.execute().get()) { + Assert.assertNotNull(resp); + } + + try (ClickHouseResponse resp = req.executeAndWait()) { + Assert.assertNotNull(resp); + } + } + + try (ClickHouseResponse resp = client.connect(server).query("select count(1) from test_custom_writer") + .executeAndWait()) { + Assert.assertEquals(resp.firstRecord().getValue(0).asInteger(), i.get() - 1); + } + } + } + @Test(groups = { "integration" }) public void testDumpAndLoadFile() throws Exception { // super.testLoadRawData(); From 5a1c464887990e985e5af29146af1017e103b6f5 Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Thu, 23 Jun 2022 20:25:26 +0800 Subject: [PATCH 3/3] Bump version and update docs --- README.md | 172 +++++++----------- clickhouse-cli-client/README.md | 4 +- clickhouse-client/README.md | 55 ++++-- clickhouse-jdbc/README.md | 127 ++++++++++++- .../util/ClickHouseVersionNumberUtil.java | 2 +- 5 files changed, 228 insertions(+), 132 deletions(-) diff --git a/README.md b/README.md index 9f48f47d8..7fe72425a 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,9 @@ Java 8 or higher is required in order to use Java client([clickhouse-client](htt :exclamation: **IMPORTANT** -Maven groupId `ru.yandex.clickhouse` and legacy JDBC driver `ru.yandex.clickhouse.ClickHouseDriver` are deprecated. +Maven groupId `ru.yandex.clickhouse` and legacy JDBC driver `ru.yandex.clickhouse.ClickHouseDriver` have been deprecated and no longer receive updates. -Please use new groupId `com.clickhouse` and driver `com.clickhouse.jdbc.ClickHouseDriver` instead. It's highly recommended to upgrade to 0.3.2+ and start to integrate the new JDBC driver for improved performance and stability. +Please use new groupId `com.clickhouse` and driver `com.clickhouse.jdbc.ClickHouseDriver` instead. It's highly recommended to upgrade to 0.3.2+ now for improved performance and stability. ![image](https://user-images.githubusercontent.com/4270380/154429324-631f718d-9277-4522-b60d-13f87b2e6c31.png) Note: in general, the new driver(v0.3.2) is a few times faster with less memory usage. More information can be found at [here](https://github.com/ClickHouse/clickhouse-jdbc/issues/768). @@ -21,90 +21,36 @@ Note: in general, the new driver(v0.3.2) is a few times faster with less memory ## Features -| Category | Feature | Supported | Remark | -| ------------- | ------------------------------------------------------------ | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Protocol | [HTTP](https://clickhouse.com/docs/en/interfaces/http/) | :white_check_mark: | recommended, defaults to `java.net.HttpURLConnection` and can be changed to `java.net.http.HttpClient`(faster but less stable) | -| | [gRPC](https://clickhouse.com/docs/en/interfaces/grpc/) | :white_check_mark: | still experimental, works with 22.3+, known to has [issue](https://github.com/ClickHouse/ClickHouse/issues/28671#issuecomment-1087049993) when using LZ4 compression | -| | [TCP/Native](https://clickhouse.com/docs/en/interfaces/tcp/) | :x: | will be available in 0.3.3 | -| Compatibility | Server < 20.7 | :x: | use 0.3.1-patch(or 0.2.6 if you're stuck with JDK 7) | -| | Server >= 20.7 | :white_check_mark: | use 0.3.2 or above. All [active releases](https://github.com/ClickHouse/ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. | -| Data Type | AggregatedFunction | :x: | limited to `groupBitmap` | -| | Array(\*) | :white_check_mark: | | -| | Bool | :white_check_mark: | | -| | Date\* | :white_check_mark: | | -| | DateTime\* | :white_check_mark: | | -| | Decimal\* | :white_check_mark: | `SET output_format_decimal_trailing_zeros=1` in 21.9+ for consistency | -| | Enum\* | :white_check_mark: | can be treated as both string and integer | -| | Geo Types | :white_check_mark: | Point, Ring, Polygon, and MultiPolygon | -| | Int\*, UInt\* | :white_check_mark: | UInt64 is mapped to `long` | -| | IPv\* | :white_check_mark: | | -| | Map(\*) | :white_check_mark: | | -| | Nested(\*) | :white_check_mark: | | -| | Object('JSON') | :white_check_mark: | | -| | SimpleAggregateFunction | :white_check_mark: | | -| | \*String | :white_check_mark: | | -| | Tuple(\*) | :white_check_mark: | | -| | UUID | :white_check_mark: | | -| Format | RowBinary | :white_check_mark: | `RowBinaryWithNamesAndTypes` for query and `RowBinary` for insertion | -| | TabSeparated | :white_check_mark: | Does not support as many data types as RowBinary | - -## Configuration - -- Client option, server setting, and default value - - You can pass any client option([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java) and [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java)) to `ClickHouseRequest.option()` and [server setting](https://clickhouse.com/docs/en/operations/settings/) to `ClickHouseRequest.set()` before execution, for instance: - - ```java - ClickHouseRequest request = client.connect(myServer); - request - .query("select 1") - // short version of option(ClickHouseClientOption.FORMAT, ClickHouseFormat.RowBinaryWithNamesAndTypes) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .option(ClickHouseClientOption.SOCKET_TIMEOUT, 30000 * 2) - .set("max_rows_to_read", 100) - .set("read_overflow_mode", "throw") - .execute() - .whenComplete((response, throwable) -> { - if (throwable != null) { - log.error("Unexpected error", throwable); - } else { - try { - for (ClickHouseRecord rec : response.records()) { - // ... - } - } finally { - response.close(); - } - } - }); - ``` - - [Default value](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java) can be either configured via system property or environment variable. - -- JDBC configuration - - **Driver Class**: `com.clickhouse.jdbc.ClickHouseDriver` - - Note: `ru.yandex.clickhouse.ClickHouseDriver` and everything under `ru.yandex.clickhouse` will be removed starting from 0.4.0. - - **URL Syntax**: `jdbc:[:]://:[][/[?param1=value1¶m2=value2]]`, for examples: - - - `jdbc:ch://localhost` is same as `jdbc:clickhouse:http://localhost:8123` - - `jdbc:ch:grpc://localhost` is same as `jdbc:clickhouse:grpc://localhost:9100` - - `jdbc:ch://localhost/test?socket_timeout=120000` - - **Connection Properties**: - - | Property | Default | Description | - | -------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | - | continueBatchOnError | `false` | Whether to continue batch processing when error occurred | - | custom_http_headers | | comma separated custom http headers, for example: `User-Agent=client1,X-Gateway-Id=123` | - | custom_http_params | | comma separated custom http query parameters, for example: `extremes=0,max_result_rows=100` | - | jdbcCompliance | `true` | Whether to support standard synchronous UPDATE/DELETE and fake transaction | - | typeMappings | | Customize mapping between ClickHouse data type and Java class, which will affect result of both [getColumnType()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSetMetaData.html#getColumnType-int-) and [getObject(Class)](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-java.lang.String-java.lang.Class-). For example: `UInt128=java.lang.String,UInt256=java.lang.String` | - | wrapperObject | `false` | Whether [getObject()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-int-) should return java.sql.Array / java.sql.Struct for Array / Tuple. | - - Note: please refer to [JDBC specific configuration](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java) and client options([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java) and [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java)) for more. +| Category | Feature | Supported | Remark | +| ----------------- | -------------------------------------------------------------------- | ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| API | [JDBC](https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/) | :white_check_mark: | | +| | [R2DBC](https://r2dbc.io/) | :white_check_mark: | 1.0.0GA was supported since 0.3.2-patch10 | +| Protocol | [HTTP](https://clickhouse.com/docs/en/interfaces/http/) | :white_check_mark: | recommended, defaults to `java.net.HttpURLConnection` and can be changed to `java.net.http.HttpClient`(faster but less stable) | +| | [gRPC](https://clickhouse.com/docs/en/interfaces/grpc/) | :white_check_mark: | still experimental, works with 22.3+, known to has [issue](https://github.com/ClickHouse/ClickHouse/issues/28671#issuecomment-1087049993) when using LZ4 compression | +| | [TCP/Native](https://clickhouse.com/docs/en/interfaces/tcp/) | :white_check_mark: | `clickhouse-cli-client`(wrapper of ClickHouse native command-line client) was added in 0.3.2-patch10, `clickhouse-tcp-client` will be available in 0.3.3 | +| Compatibility | Server < 20.7 | :x: | use 0.3.1-patch(or 0.2.6 if you're stuck with JDK 7) | +| | Server >= 20.7 | :white_check_mark: | use 0.3.2 or above. All [active releases](https://github.com/ClickHouse/ClickHouse/pulls?q=is%3Aopen+is%3Apr+label%3Arelease) are supported. | +| Data Format | RowBinary | :white_check_mark: | `RowBinaryWithNamesAndTypes` for query and `RowBinary` for insertion | +| | TabSeparated | :white_check_mark: | Does not support as many data types as RowBinary | +| Data Type | AggregatedFunction | :x: | limited to `groupBitmap` | +| | Array(\*) | :white_check_mark: | | +| | Bool | :white_check_mark: | | +| | Date\* | :white_check_mark: | | +| | DateTime\* | :white_check_mark: | | +| | Decimal\* | :white_check_mark: | `SET output_format_decimal_trailing_zeros=1` in 21.9+ for consistency | +| | Enum\* | :white_check_mark: | can be treated as both string and integer | +| | Geo Types | :white_check_mark: | Point, Ring, Polygon, and MultiPolygon | +| | Int\*, UInt\* | :white_check_mark: | UInt64 is mapped to `long` | +| | IPv\* | :white_check_mark: | | +| | Map(\*) | :white_check_mark: | | +| | Nested(\*) | :white_check_mark: | | +| | Object('JSON') | :white_check_mark: | supported since 0.3.2-patch8 | +| | SimpleAggregateFunction | :white_check_mark: | | +| | \*String | :white_check_mark: | | +| | Tuple(\*) | :white_check_mark: | | +| | UUID | :white_check_mark: | | +| High Availability | Load Balancing | :white_check_mark: | supported since 0.3.2-patch10 | +| | Failover | :white_check_mark: | supported since 0.3.2-patch10 | ## Examples @@ -115,28 +61,28 @@ Note: in general, the new driver(v0.3.2) is a few times faster with less memory com.clickhouse clickhouse-http-client - 0.3.2-patch9 + 0.3.2-patch10 ``` ```java -// only HTTP and gRPC are supported at this point -ClickHouseProtocol preferredProtocol = ClickHouseProtocol.HTTP; -// you'll have to parse response manually if use different format -ClickHouseFormat preferredFormat = ClickHouseFormat.RowBinaryWithNamesAndTypes; - -// connect to localhost, use default port of the preferred protocol -ClickHouseNode server = ClickHouseNode.builder().port(preferredProtocol).build(); - -try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol); - ClickHouseResponse response = client.connect(server) - .format(preferredFormat) +// endpoint: protocol://host[:port][/database][?param1=value1¶m2=value2...][#tag1,tag2,...] +ClickHouseNode endpoint = ClickHouseNode.of("https://localhost"); // http://localhost:8443?ssl=true&sslmode=NONE +// endpoints: [defaultProtocol://]endpoint1[,endpoint2,endpoint3,...][/defaultDatabase][?defaultParameters][#efaultTags] +ClickHouseNodes endpoints = ClickHouseNodes.of("http://(https://explorer@play.clickhouse.com:443),localhost,(tcp://localhost?!auto_discovery#experimental),(grpc://localhost#experimental)?failover=3#test") + +try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.HTTP); + ClickHouseResponse response = client.connect(endpoint) // or client.connect(endpoints) + // you'll have to parse response manually if using a different format + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) .query("select * from numbers(:limit)") .params(1000).executeAndWait()) { - // or resp.stream() if you prefer stream API + // or response.stream() if you prefer stream API for (ClickHouseRecord r : response.records()) { int num = r.getValue(0).asInteger(); + // type conversion String str = r.getValue(0).asString(); + LocalDate date = r.getValue(0).asDate(); } ClickHouseResponseSummary summary = response.getSummary(); @@ -148,12 +94,12 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol); ```xml - + com.clickhouse clickhouse-jdbc - 0.3.2-patch9 - - http + 0.3.2-patch10 + + all * @@ -164,16 +110,19 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol); ``` ```java -String url = "jdbc:ch://localhost/test"; +// jdbc:(ch|clickhouse):[defaultProtocol://]endpoint1[,endpoint2,endpoint3,...][/defaultDatabase][?defaultParameters][#efaultTags] +String url = "jdbc:ch:https://play.clickhouse.com:443"; Properties properties = new Properties(); -// optionally set connection properties +properties.setProperty("user", "explorer"); +properties.setProperty("password", ""); +// optional properties properties.setProperty("client_name", "Agent #1"); ... ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); try (Connection conn = dataSource.getConnection(); Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("select * from mytable")) { + ResultSet rs = stmt.executeQuery("show databases")) { ... } ``` @@ -193,7 +142,16 @@ Use `mvn clean verify` to compile, test and generate shaded packages if you're u 11 - ${{ env.JDK11_HOME }} + /usr/lib/jvm/java-11-openjdk + + + + jdk + + 17 + + + /usr/lib/jvm/java-17-openjdk diff --git a/clickhouse-cli-client/README.md b/clickhouse-cli-client/README.md index 649444ac3..e6e388b92 100644 --- a/clickhouse-cli-client/README.md +++ b/clickhouse-cli-client/README.md @@ -18,10 +18,10 @@ Either [clickhouse-client](https://clickhouse.com/docs/en/interfaces/cli/) or [d ```xml - + com.clickhouse clickhouse-cli-client - 0.3.2-patch9 + 0.3.2-patch10 ``` diff --git a/clickhouse-client/README.md b/clickhouse-client/README.md index 47baaef82..01bca3da9 100644 --- a/clickhouse-client/README.md +++ b/clickhouse-client/README.md @@ -1,7 +1,36 @@ # ClickHouse Java Client -Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so it does not work by itself until being used together with an implementation like `clickhouse-grpc-client` or `clickhouse-http-client`. +Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so it does not work by itself until being used together with an implementation like `clickhouse-http-client`, `clickhouse-grpc-client` or `clickhouse-cli-client`. +## Configuration + +You can pass any client option([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java), [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java), and [cli](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java)) to `ClickHouseRequest.option()` and [server setting](https://clickhouse.com/docs/en/operations/settings/) to `ClickHouseRequest.set()` before execution, for instance: + +```java +client.connect("http://localhost/system") + .query("select 1") + // short version of option(ClickHouseClientOption.FORMAT, ClickHouseFormat.RowBinaryWithNamesAndTypes) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .option(ClickHouseClientOption.SOCKET_TIMEOUT, 30000 * 2) // 60 seconds + .set("max_rows_to_read", 100) + .set("read_overflow_mode", "throw") + .execute() + .whenComplete((response, throwable) -> { + if (throwable != null) { + log.error("Unexpected error", throwable); + } else { + try { + for (ClickHouseRecord rec : response.records()) { + // ... + } + } finally { + response.close(); + } + } + }); +``` + +[Default value](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseDefaults.java) can be either configured via system property or environment variable. ## Quick Start @@ -9,16 +38,18 @@ Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so com.clickhouse clickhouse-http-client - 0.3.2-patch9 + 0.3.2-patch10 ``` ```java -// declare a server to connect to -ClickHouseNode server = ClickHouseNode.of("server1.domain", ClickHouseProtocol.HTTP, 8123, "my_db"); +// declare a list of servers to connect to +ClickHouseNodes servers = ClickHouseNodes.of( + "jdbc:ch:http://server1.domain,server2.domain,server3.domain/my_db" + + "?load_balancing_policy=random&health_check_interval=5000&failover=2"); // execute multiple queries in a worker thread one after another within same session -CompletableFuture> future = ClickHouseClient.send(server, +CompletableFuture> future = ClickHouseClient.send(servers.get(), "create database if not exists test", "use test", // change current database from my_db to test "create table if not exists test_table(s String) engine=Memory", @@ -30,20 +61,18 @@ CompletableFuture> future = ClickHouseClient.sen // block current thread until queries completed, and then retrieve summaries // List results = future.get(); -try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol())) { - ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); +try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.HTTP)) { + ClickHouseRequest request = client.connect(servers).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); // load data into a table and wait until it's completed - request.write().query("insert into my_table select c2, c3 from input('c1 UInt8, c2 String, c3 Int32')") + request.write() + .query("insert into my_table select c2, c3 from input('c1 UInt8, c2 String, c3 Int32')") .data(myInputStream).execute().thenAccept(response -> { response.close(); }); // query with named parameter - try (ClickHouseResponse response = request.query( - ClickHouseParameterizedQuery.of( - request.getConfig(), - "select * from numbers(:limit)") - ).params(100000).executeAndWait()) { + try (ClickHouseResponse response = request.query(ClickHouseParameterizedQuery.of( + request.getConfig(), "select * from numbers(:limit)")).params(100000).executeAndWait()) { for (ClickHouseRecord r : response.records()) { // Don't cache ClickHouseValue / ClickHouseRecord as they're reused for // corresponding column / row diff --git a/clickhouse-jdbc/README.md b/clickhouse-jdbc/README.md index ab96c08f7..4f4a7a72b 100644 --- a/clickhouse-jdbc/README.md +++ b/clickhouse-jdbc/README.md @@ -2,16 +2,24 @@ Build on top of `clickhouse-client`, `clickhouse-jdbc` follows JDBC standards and provides additional features like custom type mapping, fake transaction, and standard synchronous UPDATE and DELETE statement etc., so that it can be easily used together with legacy applications and tools. -Keep in mind that `clickhouse-jdbc` is synchronous, and in general it has more overheads(e.g. SQL parsing and type mapping/conversion etc.). You should consider `clickhouse-client` when performance is critical and/or you prefer more direct way to work with ClickHouse. +Keep in mind that `clickhouse-jdbc` is synchronous, and in general it has more overheads(e.g. SQL parsing and type mapping/conversion etc.). You should consider `clickhouse-client` when performance is critical and/or you prefer more direct way to access ClickHouse. ## Maven Dependency ```xml - + com.clickhouse clickhouse-jdbc - 0.3.2-patch9 + 0.3.2-patch10 + + all + + + * + * + + ``` @@ -19,13 +27,13 @@ Keep in mind that `clickhouse-jdbc` is synchronous, and in general it has more o **Driver Class**: `com.clickhouse.jdbc.ClickHouseDriver` -Note: `ru.yandex.clickhouse.ClickHouseDriver` and everything under `ru.yandex.clickhouse` will be removed starting from 0.4.0. +Note: `ru.yandex.clickhouse.ClickHouseDriver` has been deprecated and everything under `ru.yandex.clickhouse` will be removed in 0.3.3. -**URL Syntax**: `jdbc:[:]://:[][/[?param1=value1¶m2=value2]]`, for examples: +**URL Syntax**: `jdbc:(ch|clickhouse)[:]://endpoint1[,endpoint2,...][/][?param1=value1¶m2=value2][#tag1,tag2,...]`, for examples: +- `jdbc:ch://localhost` is same as `jdbc:clickhouse:http://localhost:8123` +- `jdbc:ch:https://localhost` is same as `jdbc:clickhouse:http://localhost:8443?ssl=true&sslmode=STRICT` - `jdbc:ch:grpc://localhost` is same as `jdbc:clickhouse:grpc://localhost:9100` -- `jdbc:ch:grpc://localhost` is same as `jdbc:clickhouse:grpc://localhost:9100`) -- `jdbc:ch://localhost/test?socket_timeout=120000` **Connection Properties**: @@ -35,11 +43,12 @@ Note: `ru.yandex.clickhouse.ClickHouseDriver` and everything under `ru.yandex.cl | createDatabaseIfNotExist | `false` | Whether to create database if it does not exist | | custom_http_headers | | comma separated custom http headers, for example: `User-Agent=client1,X-Gateway-Id=123` | | custom_http_params | | comma separated custom http query parameters, for example: `extremes=0,max_result_rows=100` | +| nullAsDefault | `0` | `0` - treat null value as is and throw exception when inserting null into non-nullable column; `1` - treat null value as is and disable null-check for inserting; `2` - replace null to default value of corresponding data type for both query and insert | | jdbcCompliance | `true` | Whether to support standard synchronous UPDATE/DELETE and fake transaction | | typeMappings | | Customize mapping between ClickHouse data type and Java class, which will affect result of both [getColumnType()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSetMetaData.html#getColumnType-int-) and [getObject(Class)](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-java.lang.String-java.lang.Class-). For example: `UInt128=java.lang.String,UInt256=java.lang.String` | | wrapperObject | `false` | Whether [getObject()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-int-) should return java.sql.Array / java.sql.Struct for Array / Tuple. | -Note: please refer to [JDBC specific configuration](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java) and client options([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java) and [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java)) for more. +Note: please refer to [JDBC specific configuration](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java) and client options([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java), [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java), and [cli](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-cli-client/src/main/java/com/clickhouse/client/cli/config/ClickHouseCommandLineOption.java)) for more. ## Examples @@ -48,7 +57,7 @@ Note: please refer to [JDBC specific configuration](https://github.com/ClickHous ```java String url = "jdbc:ch://my-server/system"; // use http protocol and port 8123 by default -// String url = "jdbc:ch://my-server:8443/system"; // if you prefer https +// String url = "jdbc:ch://my-server:8443/system?ssl=true&sslmode=strict&&sslrootcert=/mine.crt"; Properties properties = new Properties(); // properties.setProperty("ssl", "true"); // properties.setProperty("sslmode", "NONE"); // NONE to trust all servers; STRICT for trusted only @@ -257,3 +266,103 @@ ClickHouseFormat.RowBinary); // RowBinary or Native are supported ``` + +## Upgrade to 0.3.2 + +Please refer to cheatsheet below to upgrade JDBC driver to 0.3.2. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#Item<= 0.3.1-patch>= 0.3.2
1pom.xml
<dependency>
+    <groupId>ru.yandex.clickhouse</groupId>
+    <artifactId>clickhouse-jdbc</artifactId>
+    <version>0.3.1-patch</version>
+    <classifier>shaded</classifier>
+    <exclusions>
+        <exclusion>
+            <groupId>*</groupId>
+            <artifactId>*</artifactId>
+        </exclusion>
+    </exclusions>
+</dependency>
+
<dependency>
+    <groupId>com.clickhouse</groupId>
+    <artifactId>clickhouse-jdbc</artifactId>
+    <version>0.3.2-patch10</version>
+    <classifier>all</classifier>
+    <exclusions>
+        <exclusion>
+            <groupId>*</groupId>
+            <artifactId>*</artifactId>
+        </exclusion>
+    </exclusions>
+</dependency>
+
2driver classru.yandex.clickhouse.ClickHouseDrivercom.clickhouse.jdbc.ClickHouseDriver
3connection string
jdbc:clickhouse://[user[:password]@]host:port[/database][?parameters]
jdbc:(ch|clickhouse)[:protocol]://endpoint[,endpoint][/database][?parameters][#tags]
+endpoint: [protocol://]host[:port][/database][?parameters][#tags]
+protocol: (grpc|grpcs|http|https|tcp|tcps)
+
4load balancing
String connString = "jdbc:clickhouse://server1:8123,server2:8123,server3:8123/database";
+BalancedClickhouseDataSource balancedDs = new BalancedClickhouseDataSource(
+    connString).scheduleActualization(5000, TimeUnit.MILLISECONDS);
+ClickHouseConnection conn = balancedDs.getConnection("default", "");
+
String connString = "jdbc:ch://server1,server2,server3/database"
+    + "?load_balancing_policy=random&health_check_interval=5000&failover=2";
+ClickHouseDataSource ds = new ClickHouseDataSource(connString);
+ClickHouseConnection conn = ds.getConnection("default", "");
+
5extended API
ClickHouseStatement sth = connection.createStatement();
+sth.write().send("INSERT INTO test.writer", new ClickHouseStreamCallback() {
+    @Override
+    public void writeTo(ClickHouseRowBinaryStream stream) throws IOException {
+        for (int i = 0; i < 10; i++) {
+            stream.writeInt32(i);
+            stream.writeString("Name " + i);
+        }
+    }
+}, ClickHouseFormat.RowBinary); // RowBinary or Native are supported
+
Statement sth = connection.createStatement();
+sth.unwrap(ClickHouseRequest.class).write().table("test.writer")
+    .format(ClickHouseFormat.RowBinary).data(out -> {
+    for (int i = 0; i < 10; i++) {
+        // write data into the piped stream in current thread
+        BinaryStreamUtils.writeInt32(out, i);
+        BinaryStreamUtils.writeString(out, "Name " + i);
+    }
+}).sendAndWait(); // query happens in a separate thread
+
diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java index 781544086..f1d24c5a6 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/util/ClickHouseVersionNumberUtil.java @@ -9,7 +9,7 @@ * * @deprecated As of release 0.3.2, replaced by * {@link com.clickhouse.client.ClickHouseVersion} and it will be - * removed in 0.4.0 + * removed in 0.3.3 */ @Deprecated public final class ClickHouseVersionNumberUtil {