diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 2644b40d8..00e801904 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -4,7 +4,6 @@ on: pull_request_target: branches: - master - - develop types: - opened - synchronize diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 109932927..15e8fd27d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,8 +34,14 @@ jobs: matrix: # most recent LTS releases as well as latest stable builds clickhouse: ["21.3", "21.8", "latest"] + protocol: ["http", "grpc"] + exclude: + - clickhouse: "21.3" + protocol: grpc + - clickhouse: "21.8" + protocol: grpc fail-fast: false - name: Build against ClickHouse ${{ matrix.clickhouse }} + name: Build against ClickHouse ${{ matrix.clickhouse }} (${{ matrix.protocol }}) steps: - name: Check out Git repository uses: actions/checkout@v2 @@ -82,4 +88,4 @@ jobs: ${{ runner.os }}-build- - name: Build run: | - mvn --batch-mode --update-snapshots -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} verify + mvn --batch-mode --update-snapshots -Drelease -DclickhouseVersion=${{ matrix.clickhouse }} -Dprotocol=${{ matrix.protocol }} verify diff --git a/.github/workflows/timezone.yml b/.github/workflows/timezone.yml index eb193d7b9..7909156a0 100644 --- a/.github/workflows/timezone.yml +++ b/.github/workflows/timezone.yml @@ -34,7 +34,7 @@ jobs: serverTz: ["Asia/Chongqing", "America/Los_Angeles", "Etc/UTC", "Europe/Berlin", "Europe/Moscow"] clientTz: ["Asia/Chongqing", "America/Los_Angeles", "Etc/UTC", "Europe/Berlin", "Europe/Moscow"] fail-fast: false - name: "Test TimeZones - Server: ${{ matrix.serverTz }}, Client: ${{ matrix.clientTz }}" + name: "TimeZone(C/S): ${{ matrix.clientTz }} vs. ${{ matrix.serverTz }}" steps: - name: Check out Git repository uses: actions/checkout@v2 diff --git a/README.md b/README.md index 6472dc75d..d4daa2f9f 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Java 8 or higher is required in order to use Java client([clickhouse-client](htt com.clickhouse clickhouse-http-client - 0.3.2-patch3 + 0.3.2-patch4 ``` @@ -135,7 +135,7 @@ try (ClickHouseClient client = ClickHouseClient.newInstance(preferredProtocol); com.clickhouse clickhouse-jdbc - 0.3.2-patch3 + 0.3.2-patch4 http diff --git a/clickhouse-benchmark/pom.xml b/clickhouse-benchmark/pom.xml index d6300ac0b..a7a13612f 100644 --- a/clickhouse-benchmark/pom.xml +++ b/clickhouse-benchmark/pom.xml @@ -17,9 +17,9 @@ 1.4.4 - 2.6.3 + 2.6.4 UTF-8 - 1.33 + 1.34 benchmarks diff --git a/clickhouse-client/README.md b/clickhouse-client/README.md index 9f465c47e..9cdf8bf44 100644 --- a/clickhouse-client/README.md +++ b/clickhouse-client/README.md @@ -9,7 +9,7 @@ Async Java client for ClickHouse. `clickhouse-client` is an abstract module, so com.clickhouse clickhouse-http-client - 0.3.2-patch3 + 0.3.2-patch4 ``` diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java index 1a1ff15f6..322ed6e50 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java @@ -76,6 +76,8 @@ static CompletableFuture submit(Callable task) { return (boolean) ClickHouseDefaults.ASYNC.getEffectiveDefaultValue() ? CompletableFuture.supplyAsync(() -> { try { return task.call(); + } catch (ClickHouseException e) { + throw new CompletionException(e); } catch (CompletionException e) { throw e; } catch (Exception e) { @@ -88,6 +90,8 @@ static CompletableFuture submit(Callable task) { throw new CompletionException(cause); } }, getExecutorService()) : CompletableFuture.completedFuture(task.call()); + } catch (ClickHouseException e) { + throw new CompletionException(e); } catch (CompletionException e) { throw e; } catch (Exception e) { @@ -463,9 +467,9 @@ static CompletableFuture> send(ClickHouseNode se */ static CompletableFuture> send(ClickHouseNode server, String sql, ClickHouseValue[] templates, Object[]... params) { - int len = templates == null ? 0 : templates.length; - int size = params == null ? 0 : params.length; - if (templates == null || templates.length == 0 || params == null || params.length == 0) { + int len = templates != null ? templates.length : 0; + int size = params != null ? params.length : 0; + if (len == 0 || size == 0) { throw new IllegalArgumentException("Non-empty templates and parameters are required"); } @@ -473,7 +477,7 @@ static CompletableFuture> send(ClickHouseNode se final ClickHouseNode theServer = ClickHouseCluster.probe(server); return submit(() -> { - List list = new ArrayList<>(params.length); + List list = new ArrayList<>(size); // set async to false so that we don't have to create additional thread try (ClickHouseClient client = ClickHouseClient.builder() @@ -484,13 +488,13 @@ static CompletableFuture> send(ClickHouseNode se for (int i = 0; i < size; i++) { Object[] o = params[i]; String[] arr = new String[len]; - for (int j = 0, slen = o == null ? 0 : o.length; j < slen; j++) { - if (j < len) { - arr[j] = ClickHouseValues.NULL_EXPR; - } else { - ClickHouseValue v = templates[j]; + for (int j = 0, olen = o == null ? 0 : o.length; j < len; j++) { + ClickHouseValue v = templates[j]; + if (j < olen) { arr[j] = v != null ? v.update(o[j]).toSqlExpression() : ClickHouseValues.convertToSqlExpression(o[j]); + } else { + arr[j] = v != null ? v.resetToNullOrEmpty().toSqlExpression() : ClickHouseValues.NULL_EXPR; } } try (ClickHouseResponse resp = request.params(arr).execute().get()) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java index d07bb05cf..48b229d81 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseCluster.java @@ -5,6 +5,7 @@ import java.io.Serializable; import java.net.InetSocketAddress; import java.net.Socket; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -176,7 +177,7 @@ public static ClickHouseNode probe(ClickHouseNode node, int timeout) { client.connect(address, timeout); client.setSoTimeout(timeout); OutputStream out = client.getOutputStream(); - out.write("GET /ping HTTP/1.1\r\n\r\n".getBytes()); + out.write("GET /ping HTTP/1.1\r\n\r\n".getBytes(StandardCharsets.US_ASCII)); out.flush(); byte[] buf = new byte[12]; // HTTP/1.x xxx if (client.getInputStream().read(buf) == buf.length) { diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java index 4c75e6722..6ab9cb3da 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java @@ -1213,7 +1213,7 @@ public SelfT use(String database) { checkSealed(); Object oldValue = options.put(ClickHouseClientOption.DATABASE, - ClickHouseChecker.nonBlank(database, "database")); + ClickHouseChecker.nonNull(database, "database")); if (oldValue == null || !oldValue.equals(database)) { resetCache(); } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java index 2748f2ff7..fd9639fdc 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseUtils.java @@ -30,6 +30,7 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Consumer; import java.util.function.Supplier; import java.util.function.UnaryOperator; @@ -1066,6 +1067,91 @@ public static int readEnumValues(String args, int startIndex, int len, Map readValueArray(String args, int startIndex, int len) { + List list = new LinkedList<>(); + readValueArray(args, startIndex, len, list::add); + return list.isEmpty() ? Collections.emptyList() : Collections.unmodifiableList(list); + } + + public static int readValueArray(String args, int startIndex, int len, Consumer func) { + char closeBracket = ']'; + StringBuilder builder = new StringBuilder(); + for (int i = startIndex; i < len; i++) { + char ch = args.charAt(i); + if (ch == '[') { + startIndex = i + 1; + break; + } else if (Character.isWhitespace(ch)) { + continue; + } else if (i + 1 < len) { + char nextCh = args.charAt(i + 1); + if (ch == '-' && nextCh == '-') { + i = skipSingleLineComment(args, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = skipMultiLineComment(args, i + 2, len) - 1; + } else { + startIndex = i; + break; + } + } else { + startIndex = i; + break; + } + } + + boolean hasNext = false; + for (int i = startIndex; i < len; i++) { + char ch = args.charAt(i); + if (Character.isWhitespace(ch)) { + continue; + } else if (ch == '\'') { // string + hasNext = false; + int endIndex = readNameOrQuotedString(args, i, len, builder); + func.accept(unescape(args.substring(i, endIndex))); + builder.setLength(0); + i = endIndex + 1; + } else if (ch == '[') { // array + hasNext = false; + int endIndex = skipContentsUntil(args, i + 1, len, ']'); + func.accept(args.substring(i, endIndex)); + builder.setLength(0); + i = endIndex; + } else if (ch == '(') { // tuple + hasNext = false; + int endIndex = skipContentsUntil(args, i + 1, len, ')'); + func.accept(args.substring(i, endIndex)); + builder.setLength(0); + i = endIndex; + } else if (ch == closeBracket) { + len = i + 1; + break; + } else if (ch == ',') { + hasNext = true; + String str = builder.toString(); + func.accept(str.isEmpty() || ClickHouseValues.NULL_EXPR.equalsIgnoreCase(str) ? null : str); + builder.setLength(0); + } else if (i + 1 < len) { + char nextCh = args.charAt(i + 1); + if (ch == '-' && nextCh == '-') { + i = skipSingleLineComment(args, i + 2, len) - 1; + } else if (ch == '/' && nextCh == '*') { + i = skipMultiLineComment(args, i + 2, len) - 1; + } else { + builder.append(ch); + } + } else { + builder.append(ch); + } + } + + if (hasNext || builder.length() > 0) { + String str = builder.toString(); + func.accept(str.isEmpty() || ClickHouseValues.NULL_EXPR.equalsIgnoreCase(str) ? null : str); + } + + return len; + } + public static int readParameters(String args, int startIndex, int len, List params) { char closeBracket = ')'; // startIndex points to the openning bracket Deque stack = new ArrayDeque<>(); diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java index d2d603480..7f24011d1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseVersion.java @@ -79,6 +79,19 @@ public static ClickHouseVersion of(String version) { return versionCache.get(version); } + /** + * Creates a new version object using given numbers. + * + * @param yearOrMajor year or major vrsion + * @param more more version numbers if any + * @return version + */ + public static ClickHouseVersion of(int yearOrMajor, int... more) { + int len = more != null ? more.length : 0; + return new ClickHouseVersion(false, yearOrMajor, len > 0 ? more[0] : 0, len > 1 ? more[1] : 0, + len > 2 ? more[2] : 0); + } + /** * Parses given version without caching. * @@ -460,7 +473,7 @@ public boolean check(String range) { break; } - result = nextCh == ')' ? isOlderThan(v2) : isOlderOrEqualTo(v2); + result = nextCh == ')' ? (latest || isOlderThan(v2)) : isOlderOrEqualTo(v2); if (!result) { break; } diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java index 568ce0386..64db3e867 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/ClickHouseLongValue.java @@ -12,6 +12,8 @@ * Wraper class of long. */ public class ClickHouseLongValue implements ClickHouseValue { + private static final BigInteger MASK = BigInteger.ONE.shiftLeft(Long.SIZE).subtract(BigInteger.ONE); + /** * Create a new instance representing null value of long. * @@ -135,15 +137,13 @@ public long asLong() { public BigInteger asBigInteger() { if (isNull) { return null; - } else if (!unsigned || value >= 0L) { - return BigInteger.valueOf(value); } - byte[] bytes = new byte[Long.BYTES]; - for (int i = 1; i <= Long.BYTES; i++) { - bytes[Long.BYTES - i] = (byte) ((value >>> (i * Long.BYTES)) & 0xFF); + BigInteger v = BigInteger.valueOf(value); + if (unsigned && value < 0L) { + v = v.and(MASK); } - return new BigInteger(1, bytes); + return v; } @Override @@ -256,7 +256,7 @@ public ClickHouseLongValue update(Enum value) { @Override public ClickHouseLongValue update(String value) { - return value == null ? resetToNullOrEmpty() : set(false, unsigned, Long.parseLong(value)); + return value == null ? resetToNullOrEmpty() : set(false, unsigned, new BigInteger(value).longValue()); } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java index 611185e1a..53f8d8b6d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseByteArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -403,7 +404,22 @@ public ClickHouseByteArrayValue update(Map value) { @Override public ClickHouseByteArrayValue update(String value) { - return set(new byte[] { Byte.parseByte(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_BYTE_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_BYTE_ARRAY); + } else { + byte[] arr = new byte[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Byte.parseByte(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java index cbba6fd80..c1f251f90 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseDoubleArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -399,7 +400,22 @@ public ClickHouseDoubleArrayValue update(Map value) { @Override public ClickHouseDoubleArrayValue update(String value) { - return set(new double[] { Double.parseDouble(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_DOUBLE_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_DOUBLE_ARRAY); + } else { + double[] arr = new double[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Double.parseDouble(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java index b608df867..468759ae1 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseFloatArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -399,7 +400,22 @@ public ClickHouseFloatArrayValue update(Map value) { @Override public ClickHouseFloatArrayValue update(String value) { - return set(new float[] { Float.parseFloat(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_FLOAT_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_FLOAT_ARRAY); + } else { + float[] arr = new float[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Float.parseFloat(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java index 955bc42ea..4ab644e6d 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseIntArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -399,7 +400,22 @@ public ClickHouseIntArrayValue update(Map value) { @Override public ClickHouseIntArrayValue update(String value) { - return set(new int[] { Integer.parseInt(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_INT_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_INT_ARRAY); + } else { + int[] arr = new int[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Integer.parseInt(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java index 6423b25f1..1e419aa78 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseLongArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -399,7 +400,22 @@ public ClickHouseLongArrayValue update(Map value) { @Override public ClickHouseLongArrayValue update(String value) { - return set(new long[] { Long.parseLong(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_LONG_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_LONG_ARRAY); + } else { + long[] arr = new long[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Long.parseLong(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java index 4daa591d4..9907124ef 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/data/array/ClickHouseShortArrayValue.java @@ -21,6 +21,7 @@ import java.util.Map.Entry; import com.clickhouse.client.ClickHouseChecker; +import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseValue; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.data.ClickHouseObjectValue; @@ -399,7 +400,22 @@ public ClickHouseShortArrayValue update(Map value) { @Override public ClickHouseShortArrayValue update(String value) { - return set(new short[] { Short.parseShort(value) }); + if (ClickHouseChecker.isNullOrBlank(value)) { + set(ClickHouseValues.EMPTY_SHORT_ARRAY); + } else { + List list = ClickHouseUtils.readValueArray(value, 0, value.length()); + if (list.isEmpty()) { + set(ClickHouseValues.EMPTY_SHORT_ARRAY); + } else { + short[] arr = new short[list.size()]; + int index = 0; + for (String v : list) { + arr[index++] = Short.parseShort(v); + } + set(arr); + } + } + return this; } @Override diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java index ae478f332..712fb6630 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseClusterTest.java @@ -105,8 +105,11 @@ public void testProbe() { // FIXME does not support ClickHouseProtocol.POSTGRESQL for now ClickHouseProtocol[] protocols = new ClickHouseProtocol[] { ClickHouseProtocol.GRPC, ClickHouseProtocol.HTTP, ClickHouseProtocol.MYSQL, ClickHouseProtocol.TCP }; - + ClickHouseVersion serverVersion = ClickHouseVersion.of(System.getProperty("clickhouseVersion", "latest")); for (ClickHouseProtocol p : protocols) { + if (p == ClickHouseProtocol.GRPC && !serverVersion.check("[21.1,)")) { + continue; + } ClickHouseNode node = getServer(ClickHouseProtocol.ANY, p.getDefaultPort()); ClickHouseNode probedNode = ClickHouseCluster.probe(node); Assert.assertNotEquals(probedNode, node); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java index bd99ba504..0d9ca12a9 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseUtilsTest.java @@ -34,7 +34,8 @@ public void testGetKeyValuePair() { Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(",,"), Collections.emptyMap()); Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("=,"), Collections.emptyMap()); Assert.assertEquals(ClickHouseUtils.getKeyValuePairs(" =\r ,"), Collections.emptyMap()); - Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("a\\=='b c',"), Collections.singletonMap("a=", "'b c'")); + Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("a\\=='b c',"), + Collections.singletonMap("a=", "'b c'")); Assert.assertEquals(ClickHouseUtils.getKeyValuePairs("User-Agent=New Client, X-Forward-For=1\\,2"), new HashMap() { { @@ -53,7 +54,8 @@ public void testGetLeadingComment() { Assert.assertEquals(ClickHouseUtils.getLeadingComment("-- a \r\nselect 1"), "a"); Assert.assertEquals(ClickHouseUtils.getLeadingComment(" -- a \r\n-- b\nselect 1"), "a"); Assert.assertEquals(ClickHouseUtils.getLeadingComment("/* a */select 1"), "a"); - Assert.assertEquals(ClickHouseUtils.getLeadingComment(" /* a /* b */*/ /*-- b*/\nselect 1"), "a /* b */"); + Assert.assertEquals(ClickHouseUtils.getLeadingComment(" /* a /* b */*/ /*-- b*/\nselect 1"), + "a /* b */"); Assert.assertEquals(ClickHouseUtils.getLeadingComment("select /* a */1"), ""); } @@ -113,11 +115,14 @@ public void testSkipBrackets() { Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs, 0, newArgs.length(), '('), newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = "))", 0, newArgs.length(), '('), newArgs.lastIndexOf(')')); - Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = ")]", 0, newArgs.length(), '['), newArgs.length()); - Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = "{}", 0, newArgs.length(), '{'), newArgs.length()); + Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = ")]", 0, newArgs.length(), '['), + newArgs.length()); + Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = "{}", 0, newArgs.length(), '{'), + newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = "{ '''\\'}'}", 0, newArgs.length(), '{'), newArgs.length()); - Assert.assertEquals(ClickHouseUtils.skipBrackets(newArgs = "{ -- }\n/*/*}*/*/}", 0, newArgs.length(), '{'), + Assert.assertEquals( + ClickHouseUtils.skipBrackets(newArgs = "{ -- }\n/*/*}*/*/}", 0, newArgs.length(), '{'), newArgs.length()); } @@ -133,19 +138,24 @@ public void testSkipQuotedString() { Assert.assertEquals(ClickHouseUtils.skipQuotedString(args, 2, args.length(), '\''), args.length()); String newArgs = "''''"; - Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 0, newArgs.length(), '\''), newArgs.length()); + Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 0, newArgs.length(), '\''), + newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs = "''''''", 0, newArgs.length(), '\''), newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs = "'\\''''", 0, newArgs.length(), '\''), newArgs.length()); - Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseUtils.skipQuotedString("", 0, 0, '\'')); - Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseUtils.skipQuotedString("'", 0, 1, '\'')); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseUtils.skipQuotedString("", 0, 0, '\'')); + Assert.assertThrows(IllegalArgumentException.class, + () -> ClickHouseUtils.skipQuotedString("'", 0, 1, '\'')); Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs = "''", 0, newArgs.length(), '\''), newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs = " '''\\'}'", 0, newArgs.length(), '\''), newArgs.indexOf('\\')); - Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 1, newArgs.length(), '\''), newArgs.length()); - Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 1, newArgs.length(), '\''), newArgs.length()); + Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 1, newArgs.length(), '\''), + newArgs.length()); + Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs, 1, newArgs.length(), '\''), + newArgs.length()); Assert.assertEquals(ClickHouseUtils.skipQuotedString(newArgs = "'\\\\'''", 0, newArgs.length(), '\''), newArgs.length()); } @@ -153,7 +163,8 @@ public void testSkipQuotedString() { @Test(groups = { "unit" }) public void testSkipSingleLineComment() { String args = "select 1 -- select one\n union all select 2 -- select two--"; - Assert.assertEquals(ClickHouseUtils.skipSingleLineComment(args, 11, args.length()), args.indexOf('\n') + 1); + Assert.assertEquals(ClickHouseUtils.skipSingleLineComment(args, 11, args.length()), + args.indexOf('\n') + 1); Assert.assertEquals(ClickHouseUtils.skipSingleLineComment(args, args.indexOf("--", 11), args.length()), args.length()); } @@ -161,23 +172,29 @@ public void testSkipSingleLineComment() { @Test(groups = { "unit" }) public void testSkipMultipleLineComment() { String args = "select 1 /* select 1/*one*/ -- a */, 2"; - Assert.assertEquals(ClickHouseUtils.skipMultiLineComment(args, 11, args.length()), args.lastIndexOf("*/") + 2); - Assert.assertEquals(ClickHouseUtils.skipMultiLineComment(args, 21, args.length()), args.indexOf("*/", 21) + 2); + Assert.assertEquals(ClickHouseUtils.skipMultiLineComment(args, 11, args.length()), + args.lastIndexOf("*/") + 2); + Assert.assertEquals(ClickHouseUtils.skipMultiLineComment(args, 21, args.length()), + args.indexOf("*/", 21) + 2); Assert.assertThrows(IllegalArgumentException.class, - () -> ClickHouseUtils.skipMultiLineComment(args, args.lastIndexOf("*/") + 1, args.length())); + () -> ClickHouseUtils.skipMultiLineComment(args, args.lastIndexOf("*/") + 1, + args.length())); } @Test(groups = { "unit" }) public void testSkipContentsUntilCharacters() { String args = "select 'a' as `--b`,1/*('1(/*'*/(\0*/ \0from number(10)"; - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length()), args.lastIndexOf('\0') + 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '('), args.lastIndexOf('(') + 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length()), + args.lastIndexOf('\0') + 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '('), + args.lastIndexOf('(') + 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '(', 'n'), args.lastIndexOf('n') + 1); args = "column1 AggregateFunction(quantiles(0.5, 0.9), UInt64),\ncolumn2 UInt8 not null"; Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length()), args.length()); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), ','), args.lastIndexOf(',') + 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), ','), + args.lastIndexOf(',') + 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), '@'), args.length()); } @@ -188,28 +205,38 @@ public void testSkipContentsUntilKeyword() { Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String) null, false), 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "", true), 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "", false), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), + args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", false), args.indexOf(' ')); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", true), args.indexOf('a')); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", false), args.indexOf('a')); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "From", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", true), + args.indexOf('a')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "'", false), + args.indexOf('a')); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "From", true), + args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "From", false), args.lastIndexOf("from") + 4); args = "with (SELECT 1 as a) abcb -- select\nselect abcd"; - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", true), + args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "SELECT", false), args.lastIndexOf(' ')); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", true), args.length()); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", false), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", true), + args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "abcd", false), + args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil("abcd", 0, args.length(), "abcd", true), 4); Assert.assertEquals(ClickHouseUtils.skipContentsUntil("abcd", 0, args.length(), "abcd", false), 4); args = "column1 AggregateFunction(quantiles(0.5, 0.9), UInt64),\ncolumn2 UInt64 not null"; - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", true), args.length()); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", false), args.length()); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint64", true), args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", true), + args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint128", false), + args.length()); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint64", true), + args.length()); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), "uint64", false), args.lastIndexOf("UInt64") + 6); } @@ -217,45 +244,66 @@ public void testSkipContentsUntilKeyword() { @Test(groups = { "unit" }) public void testSkipContentsUntilKeywords() { String args = "select 1 Insert, 2 as into"; - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, true), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, false), 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, true), + 1); + Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), (String[]) null, false), + 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[0], true), 1); Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[0], false), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, true), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, false), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, true), 1); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, false), 1); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, true), + 1); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { null }, false), + 1); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, true), + 1); + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "" }, false), + 1); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", "insert" }, true), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "1", "insert" }, true), args.length()); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", "insert" }, false), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "1", "insert" }, false), args.indexOf(',')); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { " ", "insert" }, true), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { " ", "insert" }, true), args.length()); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { " ", "insert" }, false), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { " ", "insert" }, false), args.length()); - Assert.assertEquals(ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, true), + Assert.assertEquals( + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, + true), args.indexOf('I')); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, false), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "1", null }, + false), args.indexOf('I')); args = "insert Into db.table(c1, c2) select d2, d3 From input('d1 String, d2 UInt8, d3 Array(UInt16)')"; Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "insert", "into" }, true), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "insert", "into" }, true), args.length()); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "insert", "into" }, false), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "insert", "into" }, false), args.indexOf('d') - 1); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "from", "input" }, true), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "from", "input" }, true), args.length()); Assert.assertEquals( - ClickHouseUtils.skipContentsUntil(args, 0, args.length(), new String[] { "from", "input" }, false), + ClickHouseUtils.skipContentsUntil(args, 0, args.length(), + new String[] { "from", "input" }, false), args.indexOf('\'') - 1); } @@ -263,7 +311,8 @@ public void testSkipContentsUntilKeywords() { public void testReadNameOrQuotedString() { String args = "123"; StringBuilder builder = new StringBuilder(); - Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args, 0, args.length(), builder), args.length()); + Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args, 0, args.length(), builder), + args.length()); Assert.assertEquals(builder.toString(), args); builder.setLength(0); @@ -272,7 +321,9 @@ public void testReadNameOrQuotedString() { Assert.assertEquals(builder.toString(), "123"); builder.setLength(0); - Assert.assertEquals(ClickHouseUtils.readNameOrQuotedString(args = " `1\"'2``3` ", 1, args.length(), builder), + Assert.assertEquals( + ClickHouseUtils.readNameOrQuotedString(args = " `1\"'2``3` ", 1, args.length(), + builder), args.lastIndexOf('`') + 1); Assert.assertEquals(builder.toString(), "1\"'2`3"); @@ -290,7 +341,8 @@ public void testReadNameOrQuotedString() { public void testReadEnumValues() { String args = "Enum( ' `''1\" ' = 1, '\\''=2 )"; Map values = new HashMap<>(); - Assert.assertEquals(ClickHouseUtils.readEnumValues(args, 5, args.length(), values), args.lastIndexOf(')') + 1); + Assert.assertEquals(ClickHouseUtils.readEnumValues(args, 5, args.length(), values), + args.lastIndexOf(')') + 1); Assert.assertEquals(values.size(), 2); Assert.assertEquals(values.get(" `'1\" "), (Integer) 1); Assert.assertEquals(values.get("'"), (Integer) 2); @@ -309,11 +361,51 @@ public void testReadEnumValues() { args = "a Enum('1)'=2), b UInt8"; values.clear(); - Assert.assertEquals(ClickHouseUtils.readEnumValues(args, 7, args.length(), values), args.lastIndexOf(',')); + Assert.assertEquals(ClickHouseUtils.readEnumValues(args, 7, args.length(), values), + args.lastIndexOf(',')); Assert.assertEquals(values.size(), 1); Assert.assertEquals(values.get("1)"), (Integer) 2); } + @Test(groups = { "unit" }) + public void testReadValueArray() { + String args = " [1, 2, 3 ] "; + List list = new LinkedList<>(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, args.indexOf('['), args.length(), list::add), + args.lastIndexOf(']') + 1); + Assert.assertEquals(list, Arrays.asList("1", "2", "3")); + + args = "['1\\'2', '2,3' , '3\n4\r5']"; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length()); + Assert.assertEquals(list, Arrays.asList("1'2", "2,3", "3\n4\r5")); + + args = "[[1,2,3],[],(),(4,5,6)]"; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length()); + Assert.assertEquals(list, Arrays.asList("[1,2,3]", "[]", "()", "(4,5,6)")); + + args = "[,null,nan,-inf,,123,]"; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length()); + Assert.assertEquals(list, Arrays.asList(null, null, "nan", "-inf", null, "123", null)); + + args = "1"; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.length()); + Assert.assertEquals(list, Arrays.asList("1")); + + args = ""; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), 0); + Assert.assertEquals(list, Collections.emptyList()); + + args = " [ ] ]"; + list.clear(); + Assert.assertEquals(ClickHouseUtils.readValueArray(args, 0, args.length(), list::add), args.indexOf(']') + 1); + Assert.assertEquals(list, Collections.emptyList()); + } + @Test(groups = { "unit" }) public void testReadParameters() { String args = "column1 AggregateFunction( quantiles(0.5, 'c \\'''([1],2) d',0.9) , UInt64),\ncolumn2 UInt8 not null"; diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java index a35d983f2..cd164943e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseVersionTest.java @@ -132,6 +132,12 @@ public void testCheckRange() { Assert.assertTrue(ClickHouseVersion.parseVersion("21.3").check("[21.3,21.4)")); Assert.assertTrue(ClickHouseVersion.parseVersion("21.8.8.29").check("[18.16,)")); + + Assert.assertFalse(ClickHouseVersion.of("").check("[18.16,)")); + Assert.assertTrue(ClickHouseVersion.of("latest").check("[18.16,)")); + + Assert.assertTrue(ClickHouseVersion.of("latest").check("(,)")); + Assert.assertTrue(ClickHouseVersion.of("latest").check("[,)")); } @Test(groups = { "unit" }) diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java new file mode 100644 index 000000000..2589bcf27 --- /dev/null +++ b/clickhouse-client/src/test/java/com/clickhouse/client/ClientIntegrationTest.java @@ -0,0 +1,951 @@ +package com.clickhouse.client; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.stream.Collectors; + +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.client.data.ClickHouseBigDecimalValue; +import com.clickhouse.client.data.ClickHouseBigIntegerValue; +import com.clickhouse.client.data.ClickHouseDateTimeValue; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.client.data.ClickHouseIntegerValue; +import com.clickhouse.client.data.ClickHouseIpv4Value; +import com.clickhouse.client.data.ClickHouseIpv6Value; +import com.clickhouse.client.data.ClickHouseLongValue; +import com.clickhouse.client.data.ClickHouseStringValue; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public abstract class ClientIntegrationTest extends BaseIntegrationTest { + protected ClickHouseResponseSummary execute(ClickHouseRequest request, String sql) throws Exception { + try (ClickHouseResponse response = request.query(sql).execute().get()) { + for (ClickHouseRecord record : response.records()) { + for (ClickHouseValue value : record) { + Assert.assertNotNull(value, "Value should never be null"); + } + } + + return response.getSummary(); + } + } + + protected abstract ClickHouseProtocol getProtocol(); + + protected abstract Class getClientClass(); + + protected ClickHouseClient getClient() { + return ClickHouseClient.newInstance(getProtocol()); + } + + protected ClickHouseNode getServer() { + return getServer(getProtocol()); + } + + @DataProvider(name = "compressionMatrix") + protected Object[][] getCompressionMatrix() { + return new Object[][] { + new Object[] { false, false }, + new Object[] { true, false }, + new Object[] { true, true }, + new Object[] { false, true } }; + } + + @DataProvider(name = "simpleTypeProvider") + protected Object[][] getSimpleTypes() { + return new Object[][] { + { ClickHouseDataType.Enum.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, + { ClickHouseDataType.Enum8.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, + { ClickHouseDataType.Enum16.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, + { ClickHouseDataType.Int8.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt8.name(), "0", "255", "1" }, + { ClickHouseDataType.Int16.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt16.name(), "0", "65535", "1" }, + { ClickHouseDataType.Int32.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt32.name(), "0", "4294967295", "1" }, + { ClickHouseDataType.Int64.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt64.name(), "0", "18446744073709551615", "1" }, + { ClickHouseDataType.Int128.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt128.name(), "0", "340282366920938463463374607431768211455", "1" }, + { ClickHouseDataType.Int256.name(), "0", "-1", "1" }, + { ClickHouseDataType.UInt256.name(), "0", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", "1" }, + { ClickHouseDataType.Float32.name(), "0.0", "-1.0", "1.0" }, + { ClickHouseDataType.Float64.name(), "0.0", "-1.0", "1.0" }, + { ClickHouseDataType.Date.name(), "1970-01-01", "1970-01-01", "1970-01-02" }, + { ClickHouseDataType.Date32.name(), "1970-01-01", "1969-12-31", "1970-01-02" }, + { ClickHouseDataType.DateTime.name(), "1970-01-01 00:00:00", "1970-01-01 00:00:00", + "1970-01-01 00:00:01" }, + { ClickHouseDataType.DateTime32.name(), "1970-01-01 00:00:00", "1970-01-01 00:00:00", + "1970-01-01 00:00:01" }, + { ClickHouseDataType.DateTime64.name() + "(3)", "1970-01-01 00:00:00", "1969-12-31 23:59:59.999", + "1970-01-01 00:00:00.001" }, + { ClickHouseDataType.Decimal.name() + "(10,9)", "0E-9", "-1.000000000", "1.000000000" }, + { ClickHouseDataType.Decimal32.name() + "(1)", "0.0", "-1.0", "1.0" }, + { ClickHouseDataType.Decimal64.name() + "(3)", "0.000", "-1.000", "1.000" }, + { ClickHouseDataType.Decimal128.name() + "(5)", "0.00000", "-1.00000", "1.00000" }, + { ClickHouseDataType.Decimal256.name() + "(7)", "0E-7", "-1.0000000", "1.0000000" }, + { ClickHouseDataType.FixedString.name() + "(3)", "0\0\0", "-1\0", "1\0\0" }, + { ClickHouseDataType.String.name(), "0", "-1", "1" }, + { ClickHouseDataType.UUID.name(), "00000000-0000-0000-0000-000000000000", + "00000000-0000-0000-ffff-ffffffffffff", "00000000-0000-0000-0000-000000000001" } }; + } + + @Test(groups = { "unit" }) + public void testInitialization() throws Exception { + Assert.assertNotNull(getProtocol(), "The client should support a non-null protocol"); + Assert.assertNotEquals(getProtocol(), ClickHouseProtocol.ANY, + "The client should support a specific protocol instead of ANY"); + + try (ClickHouseClient client1 = ClickHouseClient.builder().build(); + ClickHouseClient client2 = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false) + .build(); + ClickHouseClient client3 = ClickHouseClient.newInstance(); + ClickHouseClient client4 = ClickHouseClient.newInstance(getProtocol()); + ClickHouseClient client5 = getClient()) { + for (ClickHouseClient client : new ClickHouseClient[] { client1, client2, client3, client4, client5 }) { + Assert.assertEquals(client.getClass(), getClientClass()); + Assert.assertTrue(client.accept(getProtocol()), "The client should support protocl: " + getProtocol()); + } + } + } + + @Test(groups = { "integration" }) + public void testOpenCloseClient() throws Exception { + int count = 100; + int timeout = 3000; + ClickHouseNode server = getServer(); + for (int i = 0; i < count; i++) { + try (ClickHouseClient client = getClient(); + ClickHouseResponse response = client.connect(server).query("select 1").execute().get()) { + Assert.assertEquals(response.firstRecord().getValue(0).asInteger(), 1); + } + Assert.assertTrue(getClient().ping(server, timeout)); + } + } + + @Test(dataProvider = "compressionMatrix", groups = { "integration" }) + public void testCompression(boolean compressRequest, boolean compressResponse) throws Exception { + ClickHouseNode server = getServer(); + String uuid = UUID.randomUUID().toString(); + try (ClickHouseClient client = getClient()) { + ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); + boolean hasResult = false; + try (ClickHouseResponse resp = request + .compressServerResponse(compressResponse) + .decompressClientRequest(compressRequest) + .query("select :uuid").params(ClickHouseStringValue.of(uuid)).execute().get()) { + Assert.assertEquals(resp.firstRecord().getValue(0).asString(), uuid); + hasResult = true; + } + Assert.assertTrue(hasResult, "Should have at least one result"); + } + } + + @Test(groups = { "integration" }) + public void testFormat() throws Exception { + String sql = "select 1, 2"; + ClickHouseNode node = getServer(); + + try (ClickHouseClient client = getClient()) { + try (ClickHouseResponse response = client.connect(node) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(sql).execute().get()) { + Assert.assertEquals(response.getColumns().size(), 2); + int counter = 0; + for (ClickHouseRecord record : response.records()) { + Assert.assertEquals(record.getValue(0).asShort(), 1); + Assert.assertEquals(record.getValue(1).asShort(), 2); + counter++; + } + Assert.assertEquals(counter, 1); + } + + // now let's try again using unsupported formats + try (ClickHouseResponse response = client.connect(node).query(sql).format(ClickHouseFormat.CSV).execute() + .get()) { + String results = new BufferedReader( + new InputStreamReader(response.getInputStream(), StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + Assert.assertEquals(results, "1,2"); + } + + try (ClickHouseResponse response = client.connect(node).query(sql).format(ClickHouseFormat.JSONEachRow) + .execute().get()) { + String results = new BufferedReader( + new InputStreamReader(response.getInputStream(), StandardCharsets.UTF_8)).lines() + .collect(Collectors.joining("\n")); + Assert.assertEquals(results, "{\"1\":1,\"2\":2}"); + } + } + } + + @Test(groups = "integration") + public void testNonExistDb() throws Exception { + ClickHouseNode server = getServer(); + + try { + ClickHouseClient.send(server, "drop database non_exist_db").get(); + Assert.fail("Exception is excepted"); + } catch (ExecutionException e) { + ClickHouseException ce = ClickHouseException.of(e.getCause(), server); + Assert.assertEquals(ce.getErrorCode(), 81); + } + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).use("non_exist_db").query("select 1").execute() + .get()) { + Assert.fail("Exception is excepted"); + } catch (ExecutionException e) { + ClickHouseException ce = ClickHouseException.of(e.getCause(), server); + Assert.assertEquals(ce.getErrorCode(), 81); + } + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).use("").query("select 1").execute().get()) { + Assert.assertEquals(resp.firstRecord().getValue(0).asInteger(), 1); + } catch (Exception e) { + Assert.fail("Should not have exception"); + } + + try (ClickHouseClient client = getClient()) { + String db = new StringBuilder().append('`').append(UUID.randomUUID().toString()).append('`').toString(); + try (ClickHouseResponse resp = client.connect(server).use("").query("create database " + db).execute() + .get()) { + } + try (ClickHouseResponse resp = client.connect(server).use("").query("drop database " + db).execute() + .get()) { + } + } catch (Exception e) { + Assert.fail("Should not have exception"); + } + } + + @Test(groups = { "integration" }) + public void testQueryWithNoResult() throws Exception { + String sql = "select * from system.numbers limit 0"; + + try (ClickHouseClient client = getClient()) { + // header without row + try (ClickHouseResponse response = client.connect(getServer()) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(sql).execute().get()) { + Assert.assertEquals(response.getColumns().size(), 1); + Assert.assertNotEquals(response.getColumns(), ClickHouseDataProcessor.DEFAULT_COLUMNS); + for (ClickHouseRecord record : response.records()) { + Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); + } + } + + // no header and row + try (ClickHouseResponse response = client.connect(getServer()).format(ClickHouseFormat.RowBinary).query(sql) + .execute().get()) { + Assert.assertEquals(response.getColumns(), Collections.emptyList()); + for (ClickHouseRecord record : response.records()) { + Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); + } + } + + // custom header and row + try (ClickHouseResponse response = client.connect(getServer()).format(ClickHouseFormat.RowBinary).query(sql) + .execute().get()) { + Assert.assertEquals(response.getColumns(), Collections.emptyList()); + for (ClickHouseRecord record : response.records()) { + Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); + } + } + } + } + + @Test(groups = { "integration" }) + public void testQuery() throws Exception { + ClickHouseNode server = getServer(); + + try (ClickHouseClient client = getClient()) { + // "select * from system.data_type_families" + int limit = 10000; + String sql = "select number, toString(number) from system.numbers limit " + limit; + + try (ClickHouseResponse response = client.connect(server) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .set("send_logs_level", "trace") + .set("enable_optimize_predicate_expression", 1) + .set("log_queries_min_type", "EXCEPTION_WHILE_PROCESSING") + .query(sql).execute().get()) { + List columns = response.getColumns(); + int index = 0; + for (ClickHouseRecord record : response.records()) { + String col1 = String.valueOf(record.getValue(0).asBigInteger()); + String col2 = record.getValue(1).asString(); + Assert.assertEquals(record.size(), columns.size()); + Assert.assertEquals(col1, col2); + Assert.assertEquals(col1, String.valueOf(index++)); + } + + // int counter = 0; + // for (ClickHouseValue value : response.values()) { + // Assert.assertEquals(value.asString(), String.valueOf(index)); + // index += counter++ % 2; + // } + Assert.assertEquals(index, limit); + // Thread.sleep(30000); + /* + * while (response.hasError()) { int index = 0; for (ClickHouseColumn c : + * columns) { // RawValue v = response.getRawValue(index++); // String v = + * response.getValue(index++, String.class) } + * + * } byte[] bytes = in.readAllBytes(); String str = new String(bytes); + */ + } catch (Exception e) { + Assert.fail("Query failed", e); + } + } + } + + @Test(groups = "integration") + public void testQueryInSameThread() throws Exception { + ClickHouseNode server = getServer(); + + try (ClickHouseClient client = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false).build()) { + CompletableFuture future = client.connect(server) + .format(ClickHouseFormat.TabSeparatedWithNamesAndTypes).query("select 1,2").execute(); + // Assert.assertTrue(future instanceof ClickHouseImmediateFuture); + Assert.assertTrue(future.isDone()); + try (ClickHouseResponse resp = future.get()) { + Assert.assertEquals(resp.getColumns().size(), 2); + for (ClickHouseRecord record : resp.records()) { + Assert.assertEquals(record.size(), 2); + Assert.assertEquals(record.getValue(0).asInteger(), 1); + Assert.assertEquals(record.getValue(1).asInteger(), 2); + } + + // ClickHouseResponseSummary summary = resp.getSummary(); + // Assert.assertEquals(summary.getStatistics().getRows(), 1); + } + } + } + + @Test(groups = { "integration" }) + public void testMutation() throws Exception { + ClickHouseNode node = getServer(); + + try (ClickHouseClient client = getClient()) { + ClickHouseRequest request = client.connect(node) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .set("send_logs_level", "trace") + .set("enable_optimize_predicate_expression", 1) + .set("log_queries_min_type", "EXCEPTION_WHILE_PROCESSING"); + execute(request, "drop table if exists test_mutation;"); + execute(request, "create table if not exists test_mutation(a String, b UInt32) engine = Memory;"); + execute(request, "insert into test_mutation values('a', 1)('b', 2)"); + } + } + + @Test(groups = "integration") + public void testQueryIntervalTypes() throws Exception { + ClickHouseNode server = getServer(); + + try (ClickHouseClient client = getClient()) { + for (ClickHouseDataType type : new ClickHouseDataType[] { ClickHouseDataType.IntervalYear, + ClickHouseDataType.IntervalQuarter, ClickHouseDataType.IntervalMonth, + ClickHouseDataType.IntervalWeek, ClickHouseDataType.IntervalDay, ClickHouseDataType.IntervalHour, + ClickHouseDataType.IntervalMinute, ClickHouseDataType.IntervalSecond }) { + try (ClickHouseResponse resp = client.connect(server) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query(ClickHouseUtils.format( + "select to%1$s(0), to%1$s(-1), to%1$s(1), to%1$s(%2$d), to%1$s(%3$d)", type.name(), + Long.MIN_VALUE, Long.MAX_VALUE)) + .execute().get()) { + List records = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + records.add(record); + } + + Assert.assertEquals(records.size(), 1); + ClickHouseRecord r = records.get(0); + Assert.assertEquals(r.getValue(0).asString(), "0"); + Assert.assertEquals(r.getValue(1).asString(), "-1"); + Assert.assertEquals(r.getValue(2).asString(), "1"); + Assert.assertEquals(r.getValue(3).asString(), String.valueOf(Long.MIN_VALUE)); + Assert.assertEquals(r.getValue(4).asString(), String.valueOf(Long.MAX_VALUE)); + } + } + } + } + + @Test(groups = "integration") + public void testReadWriteDateTimeTypes() throws Exception { + ClickHouseNode server = getServer(); + + ClickHouseClient.send(server, "drop table if exists test_datetime_types", + "create table test_datetime_types(no UInt8, d0 DateTime32, d1 DateTime64(5), d2 DateTime(3)) engine=Memory") + .get(); + ClickHouseClient.send(server, "insert into test_datetime_types values(:no, :d0, :d1, :d2)", + new ClickHouseValue[] { ClickHouseIntegerValue.ofNull(), + ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE), + ClickHouseDateTimeValue.ofNull(3, ClickHouseValues.UTC_TIMEZONE), + ClickHouseDateTimeValue.ofNull(9, ClickHouseValues.UTC_TIMEZONE) }, + new Object[] { 0, "1970-01-01 00:00:00", "1970-01-01 00:00:00.123456", + "1970-01-01 00:00:00.123456789" }, + new Object[] { 1, -1, -1, -1 }, new Object[] { 2, 1, 1, 1 }, new Object[] { 3, 2.1, 2.1, 2.1 }).get(); + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * except(no) from test_datetime_types order by no").execute().get()) { + List list = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + list.add(record); + } + + Assert.assertEquals(list.size(), 4); + } + } + + @Test(groups = "integration") + public void testReadWriteDomains() throws Exception { + ClickHouseNode server = getServer(); + + ClickHouseClient.send(server, "drop table if exists test_domain_types", + "create table test_domain_types(no UInt8, ipv4 IPv4, nipv4 Nullable(IPv4), ipv6 IPv6, nipv6 Nullable(IPv6)) engine=Memory") + .get(); + + ClickHouseClient.send(server, "insert into test_domain_types values(:no, :i0, :i1, :i2, :i3)", + new ClickHouseValue[] { ClickHouseIntegerValue.ofNull(), ClickHouseIpv4Value.ofNull(), + ClickHouseIpv4Value.ofNull(), ClickHouseIpv6Value.ofNull(), ClickHouseIpv6Value.ofNull() }, + new Object[] { 0, + (Inet4Address) InetAddress.getByAddress(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0 }), + null, + Inet6Address.getByAddress(null, + new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, + (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, + (byte) 0 }, + null), + null }, + new Object[] { 1, + (Inet4Address) InetAddress.getByAddress(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 1 }), + (Inet4Address) InetAddress + .getByAddress(new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }), + Inet6Address.getByAddress(null, + new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, + (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, + (byte) 1 }, + null), + Inet6Address.getByAddress(null, + new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, + (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }, + null) }) + .get(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * except(no) from test_domain_types order by no").execute().get()) { + List list = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + list.add(record); + } + + Assert.assertEquals(list.size(), 2); + } + } + + @Test(groups = "integration") + public void testReadWriteGeoTypes() throws Exception { + ClickHouseNode server = getServer(); + + ClickHouseClient.send(server, "set allow_experimental_geo_types=1", "drop table if exists test_geo_types", + "create table test_geo_types(no UInt8, p Point, r Ring, pg Polygon, mp MultiPolygon) engine=Memory") + .get(); + + // write + ClickHouseClient.send(server, + "insert into test_geo_types values(0, (0,0), " + "[(0,0),(0,0)], [[(0,0),(0,0)],[(0,0),(0,0)]], " + + "[[[(0,0),(0,0)],[(0,0),(0,0)]],[[(0,0),(0,0)],[(0,0),(0,0)]]])", + "insert into test_geo_types values(1, (-1,-1), " + + "[(-1,-1),(-1,-1)], [[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]], " + + "[[[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]],[[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]]])", + "insert into test_geo_types values(2, (1,1), " + "[(1,1),(1,1)], [[(1,1),(1,1)],[(1,1),(1,1)]], " + + "[[[(1,1),(1,1)],[(1,1),(1,1)]],[[(1,1),(1,1)],[(1,1),(1,1)]]])") + .get(); + + // read + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * except(no) from test_geo_types order by no").execute().get()) { + List records = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + String[] values = new String[record.size()]; + int index = 0; + for (ClickHouseValue v : record) { + values[index++] = v.asString(); + } + records.add(values); + } + + Assert.assertEquals(records.size(), 3); + Assert.assertEquals(records.get(0)[0], "(0.0,0.0)"); + Assert.assertEquals(records.get(0)[1], "[(0.0,0.0),(0.0,0.0)]"); + Assert.assertEquals(records.get(0)[2], "[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]]"); + Assert.assertEquals(records.get(0)[3], + "[[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]],[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]]]"); + Assert.assertEquals(records.get(1)[0], "(-1.0,-1.0)"); + Assert.assertEquals(records.get(1)[1], "[(-1.0,-1.0),(-1.0,-1.0)]"); + Assert.assertEquals(records.get(1)[2], "[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]]"); + Assert.assertEquals(records.get(1)[3], + "[[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]],[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]]]"); + Assert.assertEquals(records.get(2)[0], "(1.0,1.0)"); + Assert.assertEquals(records.get(2)[1], "[(1.0,1.0),(1.0,1.0)]"); + Assert.assertEquals(records.get(2)[2], "[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]]"); + Assert.assertEquals(records.get(2)[3], + "[[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]],[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]]]"); + } + } + + @Test(dataProvider = "simpleTypeProvider", groups = "integration") + public void testReadWriteSimpleTypes(String dataType, String zero, String negativeOne, String positiveOne) + throws Exception { + ClickHouseNode server = getServer(); + + String typeName = dataType; + String columnName = typeName.toLowerCase(); + int currIdx = columnName.indexOf('('); + if (currIdx > 0) { + columnName = columnName.substring(0, currIdx); + } + String dropTemplate = "drop table if exists test_%s"; + String createTemplate = "create table test_%1$s(no UInt8, %1$s %2$s, n%1$s Nullable(%2$s)) engine=Memory"; + String insertTemplate = "insert into table test_%s values(%s, %s, %s)"; + + String negativeOneValue = "-1"; + String zeroValue = "0"; + String positiveOneValue = "1"; + if (dataType.startsWith(ClickHouseDataType.FixedString.name())) { + negativeOneValue = "'-1'"; + zeroValue = "'0'"; + positiveOneValue = "'1'"; + } else if (dataType.startsWith(ClickHouseDataType.UUID.name())) { + negativeOneValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(-1).asUuid()); + zeroValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(0).asUuid()); + positiveOneValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(1).asUuid()); + } + + try { + ClickHouseClient + .send(server, ClickHouseUtils.format(dropTemplate, columnName), + ClickHouseUtils.format(createTemplate, columnName, typeName), + ClickHouseUtils.format(insertTemplate, columnName, 0, zeroValue, null), + ClickHouseUtils.format(insertTemplate, columnName, 1, zeroValue, zeroValue), + ClickHouseUtils.format(insertTemplate, columnName, 2, negativeOneValue, negativeOneValue), + ClickHouseUtils.format(insertTemplate, columnName, 3, positiveOneValue, positiveOneValue)) + .get(); + } catch (ExecutionException e) { + // maybe the type is just not supported, for example: Date32 + Throwable cause = e.getCause(); + Assert.assertTrue(cause instanceof ClickHouseException); + return; + } + + ClickHouseVersion version = null; + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client + .connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(ClickHouseUtils + .format("select * except(no), version() from test_%s order by no", columnName)) + .execute().get()) { + List records = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + String[] values = new String[record.size()]; + int index = 0; + for (ClickHouseValue v : record) { + values[index++] = v.asString(); + } + records.add(values); + } + + Assert.assertEquals(records.size(), 4); + Assert.assertEquals(records.get(0)[0], zero); + Assert.assertEquals(records.get(0)[1], null); + if (version == null) { + version = ClickHouseVersion.of(records.get(0)[2]); + } + + Assert.assertEquals(records.get(1)[0], zero); + Assert.assertEquals(records.get(1)[1], zero); + Assert.assertEquals(records.get(3)[0], positiveOne); + Assert.assertEquals(records.get(3)[1], positiveOne); + + if ((ClickHouseDataType.DateTime.name().equals(dataType) + || ClickHouseDataType.DateTime32.name().equals(dataType)) && version.getMajorVersion() == 21 + && version.getMinorVersion() == 3) { + // skip DateTime and DateTime32 negative test on 21.3 since it's not doing well + // see https://github.com/ClickHouse/ClickHouse/issues/29835 for more + } else { + Assert.assertEquals(records.get(2)[0], negativeOne); + Assert.assertEquals(records.get(2)[1], negativeOne); + } + } + } + + @Test(groups = "integration") + public void testReadWriteMap() throws Exception { + ClickHouseNode server = getServer(); + + try { + ClickHouseClient + .send(server, "drop table if exists test_map_types", + "create table test_map_types(no UInt32, m Map(LowCardinality(String), Int32))engine=Memory") + .get(); + } catch (ExecutionException e) { + // looks like LowCardinality(String) as key is not supported even in 21.8 + Throwable cause = e.getCause(); + Assert.assertTrue(cause instanceof ClickHouseException); + return; + } + + // write + ClickHouseClient.send(server, "insert into test_map_types values (1, {'key1' : 1})").get(); + ClickHouseClient.send(server, "insert into test_map_types values (:n,:m)", + new String[][] { new String[] { "-1", "{'key-1' : -1}" }, new String[] { "-2", "{'key-2' : -2}" } }) + .get(); + ClickHouseClient.send(server, "insert into test_map_types values (3, :m)", + Collections.singletonMap("m", "{'key3' : 3}")).get(); + + // read + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * except(no) from test_map_types order by no").execute().get()) { + List records = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + String[] values = new String[record.size()]; + int index = 0; + for (ClickHouseValue v : record) { + values[index++] = v.asString(); + } + records.add(values); + } + + Assert.assertEquals(records.size(), 4); + } + } + + @Test(groups = "integration") + public void testReadWriteUInt64() throws Exception { + ClickHouseNode server = getServer(); + + // INSERT INTO test_table VALUES (10223372036854775100) + ClickHouseClient.send(server, "drop table if exists test_uint64_values", + "create table test_uint64_values(no UInt8, v0 UInt64, v1 UInt64, v2 UInt64, v3 UInt64) engine=Memory") + .get(); + ClickHouseClient.send(server, "insert into test_uint64_values values(:no, :v0, :v1, :v2, :v3)", + new ClickHouseValue[] { ClickHouseIntegerValue.ofNull(), + ClickHouseLongValue.ofNull(true), ClickHouseStringValue.ofNull(), + ClickHouseBigIntegerValue.ofNull(), ClickHouseBigDecimalValue.ofNull() }, + new Object[] { 0, 0L, "0", BigInteger.ZERO, BigDecimal.ZERO }, + new Object[] { 1, 1L, "1", BigInteger.ONE, BigDecimal.ONE }, + new Object[] { 2, Long.MAX_VALUE, Long.toString(Long.MAX_VALUE), BigInteger.valueOf(Long.MAX_VALUE), + BigDecimal.valueOf(Long.MAX_VALUE) }, + new Object[] { 3, -8223372036854776516L, "10223372036854775100", new BigInteger("10223372036854775100"), + new BigDecimal("10223372036854775100") }) + .get(); + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * except(no) from test_uint64_values order by no").execute().get()) { + int count = 0; + for (ClickHouseRecord r : resp.records()) { + if (count == 0) { + Assert.assertEquals(r.getValue(0).asLong(), 0L); + Assert.assertEquals(r.getValue(1).asLong(), 0L); + Assert.assertEquals(r.getValue(2).asLong(), 0L); + Assert.assertEquals(r.getValue(3).asLong(), 0L); + } else if (count == 1) { + Assert.assertEquals(r.getValue(0).asLong(), 1L); + Assert.assertEquals(r.getValue(1).asLong(), 1L); + Assert.assertEquals(r.getValue(2).asLong(), 1L); + Assert.assertEquals(r.getValue(3).asLong(), 1L); + } else if (count == 2) { + Assert.assertEquals(r.getValue(0).asLong(), Long.MAX_VALUE); + Assert.assertEquals(r.getValue(1).asLong(), Long.MAX_VALUE); + Assert.assertEquals(r.getValue(2).asLong(), Long.MAX_VALUE); + Assert.assertEquals(r.getValue(3).asLong(), Long.MAX_VALUE); + } else if (count == 3) { + Assert.assertEquals(r.getValue(0).asString(), "10223372036854775100"); + Assert.assertEquals(r.getValue(1).asBigInteger(), new BigInteger("10223372036854775100")); + Assert.assertEquals(r.getValue(2).asBigDecimal(), new BigDecimal("10223372036854775100")); + Assert.assertEquals(r.getValue(3).asLong(), -8223372036854776516L); + } + count++; + } + + Assert.assertEquals(count, 4); + } + } + + @Test(groups = "integration") + public void testQueryWithMultipleExternalTables() throws Exception { + ClickHouseNode server = getServer(); + + int tables = 30; + int rows = 10; + try (ClickHouseClient client = getClient()) { + try (ClickHouseResponse resp = client.connect(server).query("drop table if exists test_ext_data_query") + .execute().get()) { + } + + String ddl = "create table test_ext_data_query (\n" + " Cb String,\n" + " CREATETIME DateTime64(3),\n" + + " TIMESTAMP UInt64,\n" + " Cc String,\n" + " Ca1 UInt64,\n" + " Ca2 UInt64,\n" + + " Ca3 UInt64\n" + ") engine = MergeTree()\n" + "PARTITION BY toYYYYMMDD(CREATETIME)\n" + + "ORDER BY (Cb, CREATETIME, Cc);"; + try (ClickHouseResponse resp = client.connect(server).query(ddl).execute().get()) { + } + } + + String template = "avgIf(Ca1, Cb in L%1$d) as avgCa1%2$d, sumIf(Ca1, Cb in L%1$d) as sumCa1%2$d, minIf(Ca1, Cb in L%1$d) as minCa1%2$d, maxIf(Ca1, Cb in L%1$d) as maxCa1%2$d, anyIf(Ca1, Cb in L%1$d) as anyCa1%2$d, avgIf(Ca2, Cb in L%1$d) as avgCa2%2$d, sumIf(Ca2, Cb in L%1$d) as sumCa2%2$d, minIf(Ca2, Cb in L%1$d) as minCa2%2$d, maxIf(Ca2, Cb in L%1$d) as maxCa2%2$d, anyIf(Ca2, Cb in L%1$d) as anyCa2%2$d, avgIf(Ca3, Cb in L%1$d) as avgCa3%2$d, sumIf(Ca3, Cb in L%1$d) as sumCa3%2$d, minIf(Ca3, Cb in L%1$d) as minCa3%2$d, maxIf(Ca3, Cb in L%1$d) as maxCa3%2$d, anyIf(Ca3, Cb in L%1$d) as anyCa3%2$d"; + StringBuilder sql = new StringBuilder().append("select "); + List extTableList = new ArrayList<>(tables); + for (int i = 0; i < tables; i++) { + sql.append(ClickHouseUtils.format(template, i, i + 1)).append(','); + List valueList = new ArrayList<>(rows); + for (int j = i, size = i + rows; j < size; j++) { + valueList.add(String.valueOf(j)); + } + String dnExtString = String.join("\n", valueList); + InputStream inputStream = new ByteArrayInputStream(dnExtString.getBytes(StandardCharsets.UTF_8)); + ClickHouseExternalTable extTable = ClickHouseExternalTable.builder().name("L" + i).content(inputStream) + .addColumn("Cb", "String").build(); + extTableList.add(extTable); + } + + if (tables > 0) { + sql.deleteCharAt(sql.length() - 1); + } else { + sql.append('*'); + } + sql.append( + " from test_ext_data_query where TIMESTAMP >= 1625796480 and TIMESTAMP < 1625796540 and Cc = 'eth0'"); + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).query(sql.toString()) + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).external(extTableList).execute().get()) { + Assert.assertNotNull(resp.getColumns()); + Assert.assertTrue(tables <= 0 || resp.records().iterator().hasNext()); + } + } + + @Test(groups = { "integration" }) + public void testDump() throws Exception { + ClickHouseNode server = getServer(); + + Path temp = Files.createTempFile("dump", ".tsv"); + Assert.assertEquals(Files.size(temp), 0L); + + int lines = 10000; + ClickHouseResponseSummary summary = ClickHouseClient.dump(server, "select * from system.numbers limit " + lines, + ClickHouseFormat.TabSeparated, ClickHouseCompression.NONE, temp.toString()).get(); + Assert.assertNotNull(summary); + // Assert.assertEquals(summary.getReadRows(), lines); + + int counter = 0; + for (String line : Files.readAllLines(temp)) { + Assert.assertEquals(String.valueOf(counter++), line); + } + Assert.assertEquals(counter, lines); + + Files.delete(temp); + } + + @Test(groups = { "integration" }) + public void testCustomLoad() throws Exception { + ClickHouseNode server = getServer(); + + ClickHouseClient.send(server, "drop table if exists test_custom_load", + "create table test_custom_load(n UInt32, s Nullable(String)) engine = Memory").get(); + + ClickHouseClient.load(server, "test_custom_load", ClickHouseFormat.TabSeparated, + ClickHouseCompression.NONE, new ClickHouseWriter() { + @Override + public void write(OutputStream output) throws IOException { + output.write("1\t\\N\n".getBytes(StandardCharsets.US_ASCII)); + output.write("2\t123".getBytes(StandardCharsets.US_ASCII)); + } + }).get(); + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).query("select * from test_custom_load order by n") + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).execute().get()) { + Assert.assertNotNull(resp.getColumns()); + List values = new ArrayList<>(); + for (ClickHouseRecord record : resp.records()) { + String[] arr = new String[2]; + arr[0] = record.getValue(0).asString(); + arr[1] = record.getValue(1).asString(); + values.add(arr); + } + + Assert.assertEquals(values.size(), 2); + Assert.assertEquals(values.get(0), new String[] { "1", null }); + Assert.assertEquals(values.get(1), new String[] { "2", "123" }); + } + } + + @Test(groups = { "integration" }) + public void testLoadCsv() throws Exception { + ClickHouseNode server = getServer(); + + List summaries = ClickHouseClient + .send(server, "drop table if exists test_load_csv", + "create table test_load_csv(n UInt32) engine = Memory") + .get(); + Assert.assertNotNull(summaries); + Assert.assertEquals(summaries.size(), 2); + + Path temp = Files.createTempFile("data", ".tsv"); + Assert.assertEquals(Files.size(temp), 0L); + + int lines = 10000; + StringBuilder builder = new StringBuilder(); + for (int i = 0; i < lines; i++) { + builder.append(i).append('\n'); + } + Files.write(temp, builder.toString().getBytes(StandardCharsets.US_ASCII)); + Assert.assertTrue(Files.size(temp) > 0L); + + ClickHouseResponseSummary summary = ClickHouseClient.load(server, "test_load_csv", + ClickHouseFormat.TabSeparated, ClickHouseCompression.NONE, temp.toString()).get(); + Assert.assertNotNull(summary); + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).query("select count(1) from test_load_csv").execute() + .get()) { + Assert.assertEquals(resp.firstRecord().getValue(0).asInteger(), lines); + } + + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server) + .query("select min(n), max(n), count(1), uniqExact(n) from test_load_csv") + .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).execute().get()) { + Assert.assertNotNull(resp.getColumns()); + for (ClickHouseRecord record : resp.records()) { + Assert.assertNotNull(record); + Assert.assertEquals(record.getValue(0).asLong(), 0L); + Assert.assertEquals(record.getValue(1).asLong(), lines - 1); + Assert.assertEquals(record.getValue(2).asLong(), lines); + Assert.assertEquals(record.getValue(3).asLong(), lines); + } + } finally { + Files.delete(temp); + } + } + + @Test(groups = { "integration" }) + public void testMultipleQueries() throws Exception { + ClickHouseNode server = getServer(); + try (ClickHouseClient client = getClient()) { + ClickHouseRequest req = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); + + int result1 = 1; + int result2 = 2; + ClickHouseResponse queryResp = req.copy().query("select 1").execute().get(); + + try (ClickHouseResponse resp = req.copy().query("select 2").execute().get()) { + Assert.assertEquals(resp.firstRecord().getValue(0).asInteger(), result2); + } + + result2 = 0; + for (ClickHouseRecord r : queryResp.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), result1); + result2++; + } + Assert.assertEquals(result2, 1, "Should have only one record"); + } + } + + @Test(groups = { "integration" }) + public void testExternalTableAsParameter() throws Exception { + ClickHouseNode server = getServer(); + try (ClickHouseClient client = getClient(); + ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select toString(number) as query_id from numbers(100) " + + "where query_id not in (select query_id from ext_table) limit 10") + .external(ClickHouseExternalTable.builder().name("ext_table") + .columns("query_id String, a_num Nullable(Int32)").format(ClickHouseFormat.CSV) + .content(new ByteArrayInputStream( + "\"1,2,3\",\\N\n2,333".getBytes(StandardCharsets.US_ASCII))) + .build()) + .execute().get()) { + for (ClickHouseRecord r : resp.records()) { + Assert.assertNotNull(r); + } + } + } + + @Test(groups = { "integration" }) + public void testInsertWithInputFunction() throws Exception { + ClickHouseNode server = getServer(); + ClickHouseClient.send(server, "drop table if exists test_input_function", + "create table test_input_function(name String, value Nullable(Int32))engine=Memory").get(); + + try (ClickHouseClient client = getClient()) { + // default format ClickHouseFormat.TabSeparated + ClickHouseRequest req = client.connect(server); + try (ClickHouseResponse resp = req.write().query( + "insert into test_input_function select col2, col3 from " + + "input('col1 UInt8, col2 String, col3 Int32')") + .data(new ByteArrayInputStream("1\t2\t33\n2\t3\t333".getBytes(StandardCharsets.US_ASCII))).execute() + .get()) { + + } + + List values = new ArrayList<>(); + try (ClickHouseResponse resp = req.query("select * from test_input_function").execute().get()) { + for (ClickHouseRecord r : resp.records()) { + values.add(new Object[] { r.getValue(0).asObject() }); + } + } + Assert.assertEquals(values.toArray(new Object[0][]), + new Object[][] { new Object[] { "2\t33" }, new Object[] { "3\t333" } }); + } + } + + @Test(groups = "integration") + public void testTempTable() throws Exception { + ClickHouseNode server = getServer(); + // FIXME looks like grpc has problem handling session + if (server.getProtocol() == ClickHouseProtocol.GRPC) { + return; + } + + String sessionId = UUID.randomUUID().toString(); + try (ClickHouseClient client = getClient()) { + ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinary) + .session(sessionId); + request.query("drop temporary table if exists my_temp_table").execute().get(); + request.query("create temporary table my_temp_table(a Int8)").execute().get(); + request.query("insert into my_temp_table values(2)").execute().get(); + request.write().table("my_temp_table").data(new ByteArrayInputStream(new byte[] { 3 })).execute().get(); + int count = 0; + try (ClickHouseResponse resp = request.format(ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select * from my_temp_table order by a").execute().get()) { + for (ClickHouseRecord r : resp.records()) { + Assert.assertEquals(r.getValue(0).asInteger(), count++ == 0 ? 2 : 3); + } + } + Assert.assertEquals(count, 2); + } + } +} diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java index 490666037..7be6e3f7e 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickHouseLongValueTest.java @@ -285,7 +285,8 @@ public void testUnsignedValue() throws Exception { LocalDate.ofEpochDay(-1L), // Date LocalDateTime.ofEpochSecond(-1L, 0, ZoneOffset.UTC), // DateTime LocalDateTime.ofEpochSecond(new BigDecimal(bigInt, 9).longValue(), - new BigDecimal(bigInt, 9).remainder(BigDecimal.ONE).multiply(ClickHouseValues.NANOS).intValue(), + new BigDecimal(bigInt, 9).remainder(BigDecimal.ONE) + .multiply(ClickHouseValues.NANOS).intValue(), ZoneOffset.UTC), // DateTime(9) Inet4Address.getAllByName("255.255.255.255")[0], // Inet4Address Inet6Address.getAllByName("0:0:0:0:ffff:ffff:ffff:ffff")[0], // Inet6Address @@ -301,5 +302,24 @@ public void testUnsignedValue() throws Exception { buildMap(new Object[] { 1 }, new Long[] { -1L }), // typed Map Arrays.asList(Long.valueOf(-1L)) // Tuple ); + + // try again using values greater than Long.MAX_VALUE - see issue #828 + v = ClickHouseLongValue.of(-8223372036854776516L, true); + Assert.assertEquals(v.asLong(), -8223372036854776516L); + Assert.assertEquals(v.asBigInteger(), new BigInteger("10223372036854775100")); + Assert.assertEquals(v.asBigDecimal(), new BigDecimal("10223372036854775100")); + Assert.assertEquals(v.asString(), "10223372036854775100"); + + v.update(v.asLong() - 1L); + Assert.assertEquals(v.asLong(), -8223372036854776517L); + Assert.assertEquals(v.asBigInteger(), new BigInteger("10223372036854775099")); + Assert.assertEquals(v.asBigDecimal(), new BigDecimal("10223372036854775099")); + Assert.assertEquals(v.asString(), "10223372036854775099"); + + v.update("10223372036854775101"); + Assert.assertEquals(v.asLong(), -8223372036854776515L); + Assert.assertEquals(v.asBigInteger(), new BigInteger("10223372036854775101")); + Assert.assertEquals(v.asBigDecimal(), new BigDecimal("10223372036854775101")); + Assert.assertEquals(v.asString(), "10223372036854775101"); } } diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickhouseLZ4InputStreamTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickhouseLZ4InputStreamTest.java index b136880da..3cee3c722 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickhouseLZ4InputStreamTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/ClickhouseLZ4InputStreamTest.java @@ -169,7 +169,7 @@ public void testLZ4Stream() throws IOException { try (ByteArrayOutputStream baos = new ByteArrayOutputStream(); ClickHouseLZ4OutputStream outputStream = new ClickHouseLZ4OutputStream(baos, 1024 * 1024)) { for (int i = 0; i < 100000; i++) { - outputStream.write(("test" + i).getBytes()); + outputStream.write(("test" + i).getBytes(StandardCharsets.US_ASCII)); sb.append("test").append(i); } outputStream.flush(); diff --git a/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java b/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java index 75a5abf57..6d59fa67f 100644 --- a/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java +++ b/clickhouse-client/src/test/java/com/clickhouse/client/data/JsonStreamUtilsTest.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.Reader; +import java.nio.charset.StandardCharsets; import java.util.Collections; import java.util.List; @@ -185,7 +186,8 @@ public void testReadObject() throws IOException { + "\"extremes\":{\"min\":[123],\"max\":[123]},\"rows\":1,\"rows_before_limit_at_least\":1," + "\"statistics\":{\"elapsed\":0.0008974,\"rows_read\":1,\"bytes_read\":1}}"; - r = JsonStreamUtils.readObject(new ByteArrayInputStream(json.getBytes()), JsonCompactResponse.class); + r = JsonStreamUtils.readObject(new ByteArrayInputStream(json.getBytes(StandardCharsets.US_ASCII)), + JsonCompactResponse.class); Assert.assertNotNull(r); Assert.assertEquals(r.getMeta().size(), 1); Assert.assertEquals(r.getMeta().get(0).toString(), "Meta{name='123\', type='UInt8\'}"); diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java index 5e526261a..e6b95e7f6 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcClient.java @@ -59,7 +59,8 @@ protected static String getRequestEncoding(ClickHouseConfig config) { case GZIP: break; default: - log.warn("Unsupported algorithm [%s], change to [%s]", config.getDecompressAlgorithmForClientRequest(), + log.debug("Unsupported encoding [%s], change to [%s]", + config.getDecompressAlgorithmForClientRequest().encoding(), encoding); break; } @@ -86,7 +87,7 @@ protected static Compression getResultCompression(ClickHouseConfig config) { break; // case STREAM_GZIP: default: - log.warn("Unsupported algorithm [%s], change to [%s]", config.getDecompressAlgorithmForClientRequest(), + log.debug("Unsupported algorithm [%s], change to [%s]", config.getDecompressAlgorithmForClientRequest(), algorithm); break; } @@ -110,7 +111,11 @@ protected static QueryInfo convert(ClickHouseNode server, ClickHouseRequest r ClickHouseCredentials credentials = server.getCredentials(config); Builder builder = QueryInfo.newBuilder(); - builder.setDatabase(server.getDatabase(config)).setUserName(credentials.getUserName()) + String database = server.getDatabase(config); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + builder.setDatabase(server.getDatabase(config)); + } + builder.setUserName(credentials.getUserName()) .setPassword(credentials.getPassword()).setOutputFormat(request.getFormat().name()); Optional optionalValue = request.getSessionId(); diff --git a/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto b/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto index bd7cf72c6..8070d8ad7 100644 --- a/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto +++ b/clickhouse-grpc-client/src/main/proto/clickhouse_grpc.proto @@ -45,6 +45,10 @@ message ExternalTable { // Format of the data to insert to the external table. string format = 4; + // Compression type used to compress `data`. + // Supported values: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2. + string compression_type = 6; + // Settings for executing that insertion, applied after QueryInfo.settings. map settings = 5; } @@ -109,6 +113,25 @@ message QueryInfo { /// Controls how a ClickHouse server will compress query execution results before sending back to the client. /// If not set the compression settings from the configuration file will be used. Compression result_compression = 17; + + // Compression type for `input_data`, `output_data`, `totals` and `extremes`. + // Supported compression types: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2. + // When used for `input_data` the client is responsible to compress data before putting it into `input_data`. + // When used for `output_data` or `totals` or `extremes` the client receives compressed data and should decompress it by itself. + // In the latter case consider to specify also `compression_level`. + string compression_type = 18; + + // Compression level. + // WARNING: If it's not specified the compression level is set to zero by default which might be not the best choice for some compression types (see below). + // The compression level should be in the following range (the higher the number, the better the compression): + // none: compression level isn't used + // gzip: 0..9; 0 means no compression, 6 is recommended by default (compression level -1 also means 6) + // brotli: 0..11 + // lzma: 0..9; 6 is recommended by default + // zstd: 1..22; 3 is recommended by default (compression level 0 also means 3) + // lz4: 0..16; values < 0 mean fast acceleration + // bz2: 1..9 + int32 compression_level = 19; } enum LogsLevel { diff --git a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java index 33b2d265f..026e3a41d 100644 --- a/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java +++ b/clickhouse-grpc-client/src/test/java/com/clickhouse/client/grpc/ClickHouseGrpcClientTest.java @@ -1,725 +1,49 @@ package com.clickhouse.client.grpc; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.net.Inet4Address; -import java.net.Inet6Address; -import java.net.InetAddress; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.stream.Collectors; import org.testng.Assert; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -import com.clickhouse.client.BaseIntegrationTest; + import com.clickhouse.client.ClickHouseClient; -import com.clickhouse.client.ClickHouseCompression; import com.clickhouse.client.ClickHouseNode; import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseRecord; -import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; -import com.clickhouse.client.ClickHouseValue; -import com.clickhouse.client.ClickHouseValues; -import com.clickhouse.client.ClickHouseVersion; -import com.clickhouse.client.ClickHouseWriter; +import com.clickhouse.client.ClientIntegrationTest; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseDateTimeValue; -import com.clickhouse.client.data.ClickHouseExternalTable; -import com.clickhouse.client.data.ClickHouseIntegerValue; -import com.clickhouse.client.data.ClickHouseIpv4Value; -import com.clickhouse.client.data.ClickHouseIpv6Value; -import com.clickhouse.client.ClickHouseColumn; -import com.clickhouse.client.ClickHouseDataProcessor; -import com.clickhouse.client.ClickHouseDataType; -import com.clickhouse.client.ClickHouseException; import com.clickhouse.client.ClickHouseFormat; -public class ClickHouseGrpcClientTest extends BaseIntegrationTest { - @DataProvider(name = "simpleTypeProvider") - public Object[][] getSimpleTypes() { - return new Object[][] { - { ClickHouseDataType.Enum.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, - { ClickHouseDataType.Enum8.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, - { ClickHouseDataType.Enum16.name() + "('v-1' = -1, 'v0' = 0, 'v+1' = 1)", "v0", "v-1", "v+1" }, - { ClickHouseDataType.Int8.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt8.name(), "0", "255", "1" }, - { ClickHouseDataType.Int16.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt16.name(), "0", "65535", "1" }, - { ClickHouseDataType.Int32.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt32.name(), "0", "4294967295", "1" }, - { ClickHouseDataType.Int64.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt64.name(), "0", "18446744073709551615", "1" }, - { ClickHouseDataType.Int128.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt128.name(), "0", "340282366920938463463374607431768211455", "1" }, - { ClickHouseDataType.Int256.name(), "0", "-1", "1" }, - { ClickHouseDataType.UInt256.name(), "0", - "115792089237316195423570985008687907853269984665640564039457584007913129639935", "1" }, - { ClickHouseDataType.Float32.name(), "0.0", "-1.0", "1.0" }, - { ClickHouseDataType.Float64.name(), "0.0", "-1.0", "1.0" }, - { ClickHouseDataType.Date.name(), "1970-01-01", "1970-01-01", "1970-01-02" }, - { ClickHouseDataType.Date32.name(), "1970-01-01", "1969-12-31", "1970-01-02" }, - { ClickHouseDataType.DateTime.name(), "1970-01-01 00:00:00", "1970-01-01 00:00:00", - "1970-01-01 00:00:01" }, - { ClickHouseDataType.DateTime32.name(), "1970-01-01 00:00:00", "1970-01-01 00:00:00", - "1970-01-01 00:00:01" }, - { ClickHouseDataType.DateTime64.name() + "(3)", "1970-01-01 00:00:00", "1969-12-31 23:59:59.999", - "1970-01-01 00:00:00.001" }, - { ClickHouseDataType.Decimal.name() + "(10,9)", "0E-9", "-1.000000000", "1.000000000" }, - { ClickHouseDataType.Decimal32.name() + "(1)", "0.0", "-1.0", "1.0" }, - { ClickHouseDataType.Decimal64.name() + "(3)", "0.000", "-1.000", "1.000" }, - { ClickHouseDataType.Decimal128.name() + "(5)", "0.00000", "-1.00000", "1.00000" }, - { ClickHouseDataType.Decimal256.name() + "(7)", "0E-7", "-1.0000000", "1.0000000" }, - { ClickHouseDataType.FixedString.name() + "(3)", "0\0\0", "-1\0", "1\0\0" }, - { ClickHouseDataType.String.name(), "0", "-1", "1" }, - { ClickHouseDataType.UUID.name(), "00000000-0000-0000-0000-000000000000", - "00000000-0000-0000-ffff-ffffffffffff", "00000000-0000-0000-0000-000000000001" } }; - } - - private void execute(ClickHouseRequest request, String sql) throws Exception { - try (ClickHouseResponse response = request.query(sql).execute().get()) { - for (ClickHouseRecord record : response.records()) { - for (ClickHouseValue value : record) { - Assert.assertNotNull(value); - } - } - } - } - - @Test(groups = { "unit" }) - public void testInit() throws Exception { - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC)) { - Assert.assertTrue(client instanceof ClickHouseGrpcClient); - } - } - - @Test(groups = { "integration" }) - public void testOpenCloseConnection() throws Exception { - for (int i = 0; i < 100; i++) { - Assert.assertTrue(ClickHouseClient.newInstance(ClickHouseProtocol.GRPC) - .ping(getServer(ClickHouseProtocol.GRPC), 3000)); - } - } - - @Test(groups = { "integration" }) - public void testQueryWithNoResult() throws Exception { - String sql = "select * from system.numbers limit 0"; - - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - // header without row - try (ClickHouseResponse response = client.connect(getServer(ClickHouseProtocol.GRPC)) - .option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(sql).execute().get()) { - Assert.assertEquals(response.getColumns().size(), 1); - Assert.assertNotEquals(response.getColumns(), ClickHouseDataProcessor.DEFAULT_COLUMNS); - for (ClickHouseRecord record : response.records()) { - Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); - } - } - - // no header and row - try (ClickHouseResponse response = client.connect(getServer(ClickHouseProtocol.GRPC)) - .option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8).format(ClickHouseFormat.RowBinary).query(sql) - .execute().get()) { - Assert.assertEquals(response.getColumns(), Collections.emptyList()); - for (ClickHouseRecord record : response.records()) { - Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); - } - } - - // custom header and row - try (ClickHouseResponse response = client.connect(getServer(ClickHouseProtocol.GRPC)) - .option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8).format(ClickHouseFormat.RowBinary).query(sql) - .execute().get()) { - Assert.assertEquals(response.getColumns(), Collections.emptyList()); - for (ClickHouseRecord record : response.records()) { - Assert.fail(ClickHouseUtils.format("Should have no record, but we got: %s", record)); - } - } - } - } - - @Test(groups = { "integration" }) - public void testQuery() throws Exception { - ClickHouseNode node = getServer(ClickHouseProtocol.GRPC); - - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - Assert.assertTrue(client instanceof ClickHouseGrpcClient); - - // "select * from system.data_type_families" - int limit = 10000; - String sql = "select number, toString(number) from system.numbers limit " + limit; - - try (ClickHouseResponse response = client.connect(node).option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).set("send_logs_level", "trace") - .set("enable_optimize_predicate_expression", 1) - .set("log_queries_min_type", "EXCEPTION_WHILE_PROCESSING").query(sql).execute().get()) { - List columns = response.getColumns(); - int index = 0; - for (ClickHouseRecord record : response.records()) { - String col1 = String.valueOf(record.getValue(0).asBigInteger()); - String col2 = record.getValue(1).asString(); - Assert.assertEquals(record.size(), columns.size()); - Assert.assertEquals(col1, col2); - Assert.assertEquals(col1, String.valueOf(index++)); - } - - // int counter = 0; - // for (ClickHouseValue value : response.values()) { - // Assert.assertEquals(value.asString(), String.valueOf(index)); - // index += counter++ % 2; - // } - Assert.assertEquals(index, limit); - // Thread.sleep(30000); - /* - * while (response.hasError()) { int index = 0; for (ClickHouseColumn c : - * columns) { // RawValue v = response.getRawValue(index++); // String v = - * response.getValue(index++, String.class) } - * - * } byte[] bytes = in.readAllBytes(); String str = new String(bytes); - */ - } catch (Exception e) { - Assert.fail("Query failed", e); - } - } - } - - @Test(groups = "integration") - public void testQueryInSameThread() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - try (ClickHouseClient client = ClickHouseClient.builder().option(ClickHouseClientOption.ASYNC, false).build()) { - CompletableFuture future = client.connect(server) - .format(ClickHouseFormat.TabSeparatedWithNamesAndTypes).query("select 1,2").execute(); - // Assert.assertTrue(future instanceof ClickHouseImmediateFuture); - Assert.assertTrue(future.isDone()); - try (ClickHouseResponse resp = future.get()) { - Assert.assertEquals(resp.getColumns().size(), 2); - for (ClickHouseRecord record : resp.records()) { - Assert.assertEquals(record.size(), 2); - Assert.assertEquals(record.getValue(0).asInteger(), 1); - Assert.assertEquals(record.getValue(1).asInteger(), 2); - } - - ClickHouseResponseSummary summary = resp.getSummary(); - Assert.assertEquals(summary.getStatistics().getRows(), 1); - } - } - } - - @Test(groups = "integration") - public void testQueryIntervalTypes() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC)) { - for (ClickHouseDataType type : new ClickHouseDataType[] { ClickHouseDataType.IntervalYear, - ClickHouseDataType.IntervalQuarter, ClickHouseDataType.IntervalMonth, - ClickHouseDataType.IntervalWeek, ClickHouseDataType.IntervalDay, ClickHouseDataType.IntervalHour, - ClickHouseDataType.IntervalMinute, ClickHouseDataType.IntervalSecond }) { - try (ClickHouseResponse resp = client.connect(server) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query(ClickHouseUtils.format( - "select to%1$s(0), to%1$s(-1), to%1$s(1), to%1$s(%2$d), to%1$s(%3$d)", type.name(), - Long.MIN_VALUE, Long.MAX_VALUE)) - .execute().get()) { - List records = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - records.add(record); - } - - Assert.assertEquals(records.size(), 1); - ClickHouseRecord r = records.get(0); - Assert.assertEquals(r.getValue(0).asString(), "0"); - Assert.assertEquals(r.getValue(1).asString(), "-1"); - Assert.assertEquals(r.getValue(2).asString(), "1"); - Assert.assertEquals(r.getValue(3).asString(), String.valueOf(Long.MIN_VALUE)); - Assert.assertEquals(r.getValue(4).asString(), String.valueOf(Long.MAX_VALUE)); - } - } - } +public class ClickHouseGrpcClientTest extends ClientIntegrationTest { + @Override + protected ClickHouseProtocol getProtocol() { + return ClickHouseProtocol.GRPC; } - @Test(groups = "integration") - public void testReadWriteDateTimeTypes() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - ClickHouseClient.send(server, "drop table if exists test_datetime_types", - "create table test_datetime_types(no UInt8, d0 DateTime32, d1 DateTime64(5), d2 DateTime(3)) engine=Memory") - .get(); - ClickHouseClient.send(server, "insert into test_datetime_types values(:no, :d0, :d1, :d2)", - new ClickHouseValue[] { ClickHouseIntegerValue.ofNull(), - ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE), - ClickHouseDateTimeValue.ofNull(3, ClickHouseValues.UTC_TIMEZONE), - ClickHouseDateTimeValue.ofNull(9, ClickHouseValues.UTC_TIMEZONE) }, - new Object[] { 0, "1970-01-01 00:00:00", "1970-01-01 00:00:00.123456", - "1970-01-01 00:00:00.123456789" }, - new Object[] { 1, -1, -1, -1 }, new Object[] { 2, 1, 1, 1 }, new Object[] { 3, 2.1, 2.1, 2.1 }).get(); - - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select * except(no) from test_datetime_types order by no").execute().get()) { - List list = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - list.add(record); - } - - Assert.assertEquals(list.size(), 4); - } - } - - @Test(groups = "integration") - public void testDropNonExistDb() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - try { - ClickHouseClient.send(server, "drop database non_exist_db").get(); - Assert.fail("Exception is excepted"); - } catch (ExecutionException e) { - ClickHouseException ce = (ClickHouseException) e.getCause(); - Assert.assertEquals(ce.getErrorCode(), 81); - } - } - - @Test(groups = "integration") - public void testReadWriteDomains() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - ClickHouseClient.send(server, "drop table if exists test_domain_types", - "create table test_domain_types(no UInt8, ipv4 IPv4, nipv4 Nullable(IPv4), ipv6 IPv6, nipv6 Nullable(IPv6)) engine=Memory") - .get(); - - ClickHouseClient.send(server, "insert into test_domain_types values(:no, :i0, :i1, :i2, :i3)", - new ClickHouseValue[] { ClickHouseIntegerValue.ofNull(), ClickHouseIpv4Value.ofNull(), - ClickHouseIpv4Value.ofNull(), ClickHouseIpv6Value.ofNull(), ClickHouseIpv6Value.ofNull() }, - new Object[] { 0, - (Inet4Address) InetAddress.getByAddress(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0 }), - null, - Inet6Address.getByAddress(null, - new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, - (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, - (byte) 0 }, - null), - null }, - new Object[] { 1, - (Inet4Address) InetAddress.getByAddress(new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 1 }), - (Inet4Address) InetAddress - .getByAddress(new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }), - Inet6Address.getByAddress(null, - new byte[] { (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, - (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0, - (byte) 1 }, - null), - Inet6Address.getByAddress(null, - new byte[] { (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, - (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, - (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF }, - null) }) - .get(); - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select * except(no) from test_domain_types order by no").execute().get()) { - List list = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - list.add(record); - } - - Assert.assertEquals(list.size(), 2); - } + @Override + protected Class getClientClass() { + return ClickHouseGrpcClient.class; } @Test(groups = "integration") - public void testReadWriteGeoTypes() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - ClickHouseClient.send(server, "set allow_experimental_geo_types=1", "drop table if exists test_geo_types", - "create table test_geo_types(no UInt8, p Point, r Ring, pg Polygon, mp MultiPolygon) engine=Memory") - .get(); - - // write - ClickHouseClient.send(server, - "insert into test_geo_types values(0, (0,0), " + "[(0,0),(0,0)], [[(0,0),(0,0)],[(0,0),(0,0)]], " - + "[[[(0,0),(0,0)],[(0,0),(0,0)]],[[(0,0),(0,0)],[(0,0),(0,0)]]])", - "insert into test_geo_types values(1, (-1,-1), " - + "[(-1,-1),(-1,-1)], [[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]], " - + "[[[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]],[[(-1,-1),(-1,-1)],[(-1,-1),(-1,-1)]]])", - "insert into test_geo_types values(2, (1,1), " + "[(1,1),(1,1)], [[(1,1),(1,1)],[(1,1),(1,1)]], " - + "[[[(1,1),(1,1)],[(1,1),(1,1)]],[[(1,1),(1,1)],[(1,1),(1,1)]]])") - .get(); - - // read - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select * except(no) from test_geo_types order by no").execute().get()) { - List records = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - String[] values = new String[record.size()]; - int index = 0; - for (ClickHouseValue v : record) { - values[index++] = v.asString(); - } - records.add(values); - } - - Assert.assertEquals(records.size(), 3); - Assert.assertEquals(records.get(0)[0], "(0.0,0.0)"); - Assert.assertEquals(records.get(0)[1], "[(0.0,0.0),(0.0,0.0)]"); - Assert.assertEquals(records.get(0)[2], "[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]]"); - Assert.assertEquals(records.get(0)[3], - "[[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]],[[(0.0,0.0),(0.0,0.0)],[(0.0,0.0),(0.0,0.0)]]]"); - Assert.assertEquals(records.get(1)[0], "(-1.0,-1.0)"); - Assert.assertEquals(records.get(1)[1], "[(-1.0,-1.0),(-1.0,-1.0)]"); - Assert.assertEquals(records.get(1)[2], "[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]]"); - Assert.assertEquals(records.get(1)[3], - "[[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]],[[(-1.0,-1.0),(-1.0,-1.0)],[(-1.0,-1.0),(-1.0,-1.0)]]]"); - Assert.assertEquals(records.get(2)[0], "(1.0,1.0)"); - Assert.assertEquals(records.get(2)[1], "[(1.0,1.0),(1.0,1.0)]"); - Assert.assertEquals(records.get(2)[2], "[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]]"); - Assert.assertEquals(records.get(2)[3], - "[[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]],[[(1.0,1.0),(1.0,1.0)],[(1.0,1.0),(1.0,1.0)]]]"); - } - } - - @Test(dataProvider = "simpleTypeProvider", groups = "integration") - public void testReadWriteSimpleTypes(String dataType, String zero, String negativeOne, String positiveOne) - throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - String typeName = dataType; - String columnName = typeName.toLowerCase(); - int currIdx = columnName.indexOf('('); - if (currIdx > 0) { - columnName = columnName.substring(0, currIdx); - } - String dropTemplate = "drop table if exists test_%s"; - String createTemplate = "create table test_%1$s(no UInt8, %1$s %2$s, n%1$s Nullable(%2$s)) engine=Memory"; - String insertTemplate = "insert into table test_%s values(%s, %s, %s)"; - - String negativeOneValue = "-1"; - String zeroValue = "0"; - String positiveOneValue = "1"; - if (dataType.startsWith(ClickHouseDataType.FixedString.name())) { - negativeOneValue = "'-1'"; - zeroValue = "'0'"; - positiveOneValue = "'1'"; - } else if (dataType.startsWith(ClickHouseDataType.UUID.name())) { - negativeOneValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(-1).asUuid()); - zeroValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(0).asUuid()); - positiveOneValue = ClickHouseUtils.format("'%s'", ClickHouseIntegerValue.of(1).asUuid()); - } + public void testResponseSummary() throws Exception { + ClickHouseNode server = getServer(); - try { - ClickHouseClient - .send(server, ClickHouseUtils.format(dropTemplate, columnName), - ClickHouseUtils.format(createTemplate, columnName, typeName), - ClickHouseUtils.format(insertTemplate, columnName, 0, zeroValue, null), - ClickHouseUtils.format(insertTemplate, columnName, 1, zeroValue, zeroValue), - ClickHouseUtils.format(insertTemplate, columnName, 2, negativeOneValue, negativeOneValue), - ClickHouseUtils.format(insertTemplate, columnName, 3, positiveOneValue, positiveOneValue)) - .get(); - } catch (ExecutionException e) { - // maybe the type is just not supported, for example: Date32 - Throwable cause = e.getCause(); - Assert.assertTrue(cause instanceof ClickHouseException); - return; - } - - ClickHouseVersion version = null; - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - ClickHouseResponse resp = client - .connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(ClickHouseUtils - .format("select * except(no), version() from test_%s order by no", columnName)) - .execute().get()) { - List records = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - String[] values = new String[record.size()]; - int index = 0; - for (ClickHouseValue v : record) { - values[index++] = v.asString(); - } - records.add(values); - } - - Assert.assertEquals(records.size(), 4); - Assert.assertEquals(records.get(0)[0], zero); - Assert.assertEquals(records.get(0)[1], null); - if (version == null) { - version = ClickHouseVersion.of(records.get(0)[2]); - } - - Assert.assertEquals(records.get(1)[0], zero); - Assert.assertEquals(records.get(1)[1], zero); - Assert.assertEquals(records.get(3)[0], positiveOne); - Assert.assertEquals(records.get(3)[1], positiveOne); - - if ((ClickHouseDataType.DateTime.name().equals(dataType) - || ClickHouseDataType.DateTime32.name().equals(dataType)) && version.getMajorVersion() == 21 - && version.getMinorVersion() == 3) { - // skip DateTime and DateTime32 negative test on 21.3 since it's not doing well - // see https://github.com/ClickHouse/ClickHouse/issues/29835 for more - } else { - Assert.assertEquals(records.get(2)[0], negativeOne); - Assert.assertEquals(records.get(2)[1], negativeOne); - } - } - } - - @Test(groups = "integration") - public void testReadWriteMap() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - try { - ClickHouseClient - .send(server, "drop table if exists test_map_types", - "create table test_map_types(no UInt32, m Map(LowCardinality(String), Int32))engine=Memory") - .get(); - } catch (ExecutionException e) { - // looks like LowCardinality(String) as key is not supported even in 21.8 - Throwable cause = e.getCause(); - Assert.assertTrue(cause instanceof ClickHouseException); - return; - } - - // write - ClickHouseClient.send(server, "insert into test_map_types values (1, {'key1' : 1})").get(); - ClickHouseClient.send(server, "insert into test_map_types values (:n,:m)", - new String[][] { new String[] { "-1", "{'key-1' : -1}" }, new String[] { "-2", "{'key-2' : -2}" } }) - .get(); - ClickHouseClient.send(server, "insert into test_map_types values (3, :m)", - Collections.singletonMap("m", "{'key3' : 3}")).get(); - - // read - try (ClickHouseClient client = ClickHouseClient.newInstance(ClickHouseProtocol.GRPC); - ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select * except(no) from test_map_types order by no").execute().get()) { - List records = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - String[] values = new String[record.size()]; - int index = 0; - for (ClickHouseValue v : record) { - values[index++] = v.asString(); - } - records.add(values); - } - - Assert.assertEquals(records.size(), 4); - } - } - - @Test(groups = "integration") - public void testQueryWithMultipleExternalTables() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - int tables = 30; - int rows = 10; - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - try (ClickHouseResponse resp = client.connect(server).query("drop table if exists test_ext_data_query") - .execute().get()) { - } - - String ddl = "create table test_ext_data_query (\n" + " Cb String,\n" + " CREATETIME DateTime64(3),\n" - + " TIMESTAMP UInt64,\n" + " Cc String,\n" + " Ca1 UInt64,\n" + " Ca2 UInt64,\n" - + " Ca3 UInt64\n" + ") engine = MergeTree()\n" + "PARTITION BY toYYYYMMDD(CREATETIME)\n" - + "ORDER BY (Cb, CREATETIME, Cc);"; - try (ClickHouseResponse resp = client.connect(server).query(ddl).execute().get()) { - } - } - - String template = "avgIf(Ca1, Cb in L%1$d) as avgCa1%2$d, sumIf(Ca1, Cb in L%1$d) as sumCa1%2$d, minIf(Ca1, Cb in L%1$d) as minCa1%2$d, maxIf(Ca1, Cb in L%1$d) as maxCa1%2$d, anyIf(Ca1, Cb in L%1$d) as anyCa1%2$d, avgIf(Ca2, Cb in L%1$d) as avgCa2%2$d, sumIf(Ca2, Cb in L%1$d) as sumCa2%2$d, minIf(Ca2, Cb in L%1$d) as minCa2%2$d, maxIf(Ca2, Cb in L%1$d) as maxCa2%2$d, anyIf(Ca2, Cb in L%1$d) as anyCa2%2$d, avgIf(Ca3, Cb in L%1$d) as avgCa3%2$d, sumIf(Ca3, Cb in L%1$d) as sumCa3%2$d, minIf(Ca3, Cb in L%1$d) as minCa3%2$d, maxIf(Ca3, Cb in L%1$d) as maxCa3%2$d, anyIf(Ca3, Cb in L%1$d) as anyCa3%2$d"; - StringBuilder sql = new StringBuilder().append("select "); - List extTableList = new ArrayList<>(tables); - for (int i = 0; i < tables; i++) { - sql.append(ClickHouseUtils.format(template, i, i + 1)).append(','); - List valueList = new ArrayList<>(rows); - for (int j = i, size = i + rows; j < size; j++) { - valueList.add(String.valueOf(j)); - } - String dnExtString = String.join("\n", valueList); - InputStream inputStream = new ByteArrayInputStream(dnExtString.getBytes(Charset.forName("UTF-8"))); - ClickHouseExternalTable extTable = ClickHouseExternalTable.builder().name("L" + i).content(inputStream) - .addColumn("Cb", "String").build(); - extTableList.add(extTable); - } - - if (tables > 0) { - sql.deleteCharAt(sql.length() - 1); - } else { - sql.append('*'); - } - sql.append( - " from test_ext_data_query where TIMESTAMP >= 1625796480 and TIMESTAMP < 1625796540 and Cc = 'eth0'"); - - try (ClickHouseClient client = ClickHouseClient.builder().build(); - ClickHouseResponse resp = client.connect(server).query(sql.toString()) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).external(extTableList).execute().get()) { - Assert.assertNotNull(resp.getColumns()); - Assert.assertTrue(tables <= 0 || resp.records().iterator().hasNext()); - } - } - - @Test(groups = { "integration" }) - public void testMutation() throws Exception { - ClickHouseNode node = getServer(ClickHouseProtocol.GRPC); - - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - Assert.assertTrue(client instanceof ClickHouseGrpcClient); - - ClickHouseRequest request = client.connect(node).option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).set("send_logs_level", "trace") - .set("enable_optimize_predicate_expression", 1) - .set("log_queries_min_type", "EXCEPTION_WHILE_PROCESSING"); - execute(request, "drop table if exists test_grpc_mutation;"); - execute(request, "create table if not exists test_grpc_mutation(a String, b UInt32) engine = Memory;"); - execute(request, "insert into test_grpc_mutation values('a', 1)('b', 2)"); - } - } - - @Test(groups = { "integration" }) - public void testFormat() throws Exception { - String sql = "select 1, 2"; - ClickHouseNode node = getServer(ClickHouseProtocol.GRPC); - - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - try (ClickHouseResponse response = client.connect(node).option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).query(sql).execute().get()) { - Assert.assertEquals(response.getColumns().size(), 2); - int counter = 0; - for (ClickHouseRecord record : response.records()) { - Assert.assertEquals(record.getValue(0).asShort(), 1); - Assert.assertEquals(record.getValue(1).asShort(), 2); - counter++; - } - Assert.assertEquals(counter, 1); - } - - // now let's try again using unsupported formats - try (ClickHouseResponse response = client.connect(node).query(sql).format(ClickHouseFormat.CSV).execute() - .get()) { - String results = new BufferedReader( - new InputStreamReader(response.getInputStream(), StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); - Assert.assertEquals(results, "1,2"); - } - - try (ClickHouseResponse response = client.connect(node).query(sql).format(ClickHouseFormat.JSONEachRow) - .execute().get()) { - String results = new BufferedReader( - new InputStreamReader(response.getInputStream(), StandardCharsets.UTF_8)).lines() - .collect(Collectors.joining("\n")); - Assert.assertEquals(results, "{\"1\":1,\"2\":2}"); - } - } - } - - @Test(groups = { "integration" }) - public void testDump() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - Path temp = Files.createTempFile("dump", ".tsv"); - Assert.assertEquals(Files.size(temp), 0L); - - int lines = 10000; - ClickHouseResponseSummary summary = ClickHouseClient.dump(server, "select * from system.numbers limit " + lines, - ClickHouseFormat.TabSeparated, ClickHouseCompression.BROTLI, temp.toString()).get(); - Assert.assertNotNull(summary); - Assert.assertEquals(summary.getReadRows(), lines); - - int counter = 0; - for (String line : Files.readAllLines(temp)) { - Assert.assertEquals(String.valueOf(counter++), line); - } - Assert.assertEquals(counter, lines); - - Files.delete(temp); - } - - @Test(groups = { "integration" }) - public void testCustomLoad() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - ClickHouseClient.send(server, "drop table if exists test_grpc_custom_load", - "create table if not exists test_grpc_custom_load(n UInt32, s Nullable(String)) engine = Memory").get(); - - ClickHouseClient.load(server, "test_grpc_custom_load", ClickHouseFormat.TabSeparated, - ClickHouseCompression.NONE, new ClickHouseWriter() { - @Override - public void write(OutputStream output) throws IOException { - output.write("1\t\\N\n".getBytes()); - output.write("2\t123".getBytes()); - } - }).get(); - - try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol()); - ClickHouseResponse resp = client.connect(server).query("select * from test_grpc_custom_load order by n") - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).execute().get()) { - Assert.assertNotNull(resp.getColumns()); - List values = new ArrayList<>(); - for (ClickHouseRecord record : resp.records()) { - String[] arr = new String[2]; - arr[0] = record.getValue(0).asString(); - arr[1] = record.getValue(1).asString(); - values.add(arr); - } - - Assert.assertEquals(values.size(), 2); - Assert.assertEquals(values.get(0), new String[] { "1", null }); - Assert.assertEquals(values.get(1), new String[] { "2", "123" }); - } - } - - @Test(groups = { "integration" }) - public void testLoadCsv() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.GRPC); - - List summaries = ClickHouseClient - .send(server, "drop table if exists test_grpc_load_data", - "create table if not exists test_grpc_load_data(n UInt32) engine = Memory") - .get(); - Assert.assertNotNull(summaries); - Assert.assertEquals(summaries.size(), 2); - - Path temp = Files.createTempFile("data", ".tsv"); - Assert.assertEquals(Files.size(temp), 0L); - - int lines = 10000; - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < lines; i++) { - builder.append(i).append('\n'); - } - Files.write(temp, builder.toString().getBytes()); - Assert.assertTrue(Files.size(temp) > 0L); - - ClickHouseResponseSummary summary = ClickHouseClient.load(server, "test_grpc_load_data", - ClickHouseFormat.TabSeparated, ClickHouseCompression.NONE, temp.toString()).get(); - Assert.assertNotNull(summary); - Assert.assertEquals(summary.getWrittenRows(), lines); - - try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol()); + try (ClickHouseClient client = getClient(); ClickHouseResponse resp = client.connect(server) - .query("select min(n), max(n), count(1), uniqExact(n) from test_grpc_load_data") - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).execute().get()) { - Assert.assertNotNull(resp.getColumns()); + .option(ClickHouseClientOption.MAX_BUFFER_SIZE, 8) + .format(ClickHouseFormat.TabSeparatedWithNamesAndTypes) + .query("select number, number+1 from numbers(100)").execute().get()) { + int n = 0; for (ClickHouseRecord record : resp.records()) { - Assert.assertNotNull(record); - Assert.assertEquals(record.getValue(0).asLong(), 0L); - Assert.assertEquals(record.getValue(1).asLong(), lines - 1); - Assert.assertEquals(record.getValue(2).asLong(), lines); - Assert.assertEquals(record.getValue(3).asLong(), lines); + Assert.assertEquals(record.size(), 2); + Assert.assertEquals(record.getValue(0).asInteger(), n++); + Assert.assertEquals(record.getValue(1).asInteger(), n); } - } finally { - Files.delete(temp); + Assert.assertEquals(n, 100); + + ClickHouseResponseSummary summary = resp.getSummary(); + Assert.assertEquals(summary.getReadRows(), n); + Assert.assertEquals(summary.getStatistics().getRows(), n); } } } diff --git a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java index 837be1860..b2191bd5c 100644 --- a/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java +++ b/clickhouse-http-client/src/main/java/com/clickhouse/client/http/ClickHouseHttpConnection.java @@ -186,7 +186,11 @@ protected ClickHouseHttpConnection(ClickHouseNode server, ClickHouseRequest r map.put("X-ClickHouse-Key", credentials.getPassword()); } } - map.put("X-ClickHouse-Database", server.getDatabase(config)); + + String database = server.getDatabase(config); + if (!ClickHouseChecker.isNullOrEmpty(database)) { + map.put("X-ClickHouse-Database", database); + } // Also, you can use the ‘default_format’ URL parameter map.put("X-ClickHouse-Format", config.getFormat().name()); if (config.isCompressServerResponse()) { diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java index 09c9f52b5..802eb3859 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/ClickHouseHttpClientTest.java @@ -1,13 +1,7 @@ package com.clickhouse.client.http; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.List; import java.util.UUID; -import com.clickhouse.client.BaseIntegrationTest; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseCredentials; import com.clickhouse.client.ClickHouseFormat; @@ -18,126 +12,42 @@ import com.clickhouse.client.ClickHouseRequest; import com.clickhouse.client.ClickHouseResponse; import com.clickhouse.client.ClickHouseResponseSummary; -import com.clickhouse.client.ClickHouseUtils; import com.clickhouse.client.ClickHouseVersion; +import com.clickhouse.client.ClientIntegrationTest; import com.clickhouse.client.config.ClickHouseClientOption; -import com.clickhouse.client.data.ClickHouseExternalTable; import com.clickhouse.client.data.ClickHouseStringValue; import org.testng.Assert; -import org.testng.annotations.DataProvider; import org.testng.annotations.Test; -public class ClickHouseHttpClientTest extends BaseIntegrationTest { - @DataProvider(name = "compressionMatrix") - private Object[][] getCompressionMatrix() { - return new Object[][] { - new Object[] { false, false }, - new Object[] { true, false }, - new Object[] { true, true }, - new Object[] { false, true } }; +public class ClickHouseHttpClientTest extends ClientIntegrationTest { + @Override + protected ClickHouseProtocol getProtocol() { + return ClickHouseProtocol.HTTP; } - @Test(dataProvider = "compressionMatrix", groups = { "integration" }) - public void testCompression(boolean compressRequest, boolean compressResponse) throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - String uuid = UUID.randomUUID().toString(); - try (ClickHouseClient client = ClickHouseClient.newInstance()) { - ClickHouseRequest request = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); - boolean hasResult = false; - try (ClickHouseResponse resp = request - .compressServerResponse(compressResponse) - .decompressClientRequest(compressRequest) - .query("select :uuid").params(ClickHouseStringValue.of(uuid)).execute().get()) { - Assert.assertEquals(resp.firstRecord().getValue(0).asString(), uuid); - hasResult = true; - } - Assert.assertTrue(hasResult, "Should have at least one result"); - } - } - - @Test(groups = { "integration" }) - public void testPing() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - - try (ClickHouseClient client = ClickHouseClient.newInstance()) { - Assert.assertTrue(client.ping(server, 1000)); - } + @Override + protected Class getClientClass() { + return ClickHouseHttpClient.class; } @Test(groups = { "integration" }) + @Override public void testMutation() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - ClickHouseClient.send(server, "drop table if exists test_data_load", - "create table test_data_load(a String, b Nullable(Int64))engine=Memory").get(); - try (ClickHouseClient client = ClickHouseClient.newInstance(); + super.testMutation(); + + ClickHouseNode server = getServer(); + ClickHouseClient.send(server, "drop table if exists test_http_mutation", + "create table test_http_mutation(a String, b Nullable(Int64))engine=Memory").get(); + try (ClickHouseClient client = getClient(); ClickHouseResponse response = client.connect(server).set("send_progress_in_http_headers", 1) - .query("insert into test_data_load select toString(number), number from numbers(1)").execute() - .get()) { + .query("insert into test_http_mutation select toString(number), number from numbers(1)") + .execute().get()) { ClickHouseResponseSummary summary = response.getSummary(); Assert.assertEquals(summary.getWrittenRows(), 1); } } - @Test(groups = { "integration" }) - public void testMultipleQueries() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - try (ClickHouseClient client = ClickHouseClient.newInstance()) { - ClickHouseRequest req = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes); - - ClickHouseResponse queryResp = req.copy().query("select 1").execute().get(); - - try (ClickHouseResponse resp = req.copy().query("select 2").execute().get()) { - } - - for (ClickHouseRecord r : queryResp.records()) { - continue; - } - } - } - - @Test(groups = { "integration" }) - public void testExternalTableAsParameter() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - try (ClickHouseClient client = ClickHouseClient.newInstance(); - ClickHouseResponse resp = client.connect(server).format(ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select toString(number) as query_id from numbers(100) where query_id not in (select query_id from ext_table) limit 10") - .external(ClickHouseExternalTable.builder().name("ext_table") - .columns("query_id String, a_num Nullable(Int32)").format(ClickHouseFormat.CSV) - .content(new ByteArrayInputStream("\"1,2,3\",\\N\n2,333".getBytes())).build()) - .execute().get()) { - for (ClickHouseRecord r : resp.records()) { - Assert.assertNotNull(r); - } - } - } - - @Test(groups = { "integration" }) - public void testInsertWithInputFunction() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - ClickHouseClient.send(server, "drop table if exists test_input_function", - "create table test_input_function(name String, value Nullable(Int32))engine=Memory").get(); - - try (ClickHouseClient client = ClickHouseClient.newInstance()) { - // default format ClickHouseFormat.TabSeparated - ClickHouseRequest req = client.connect(server); - try (ClickHouseResponse resp = req.write().query( - "insert into test_input_function select col2, col3 from input('col1 UInt8, col2 String, col3 Int32')") - .data(new ByteArrayInputStream("1\t2\t33\n2\t3\t333".getBytes())).execute().get()) { - - } - - List values = new ArrayList<>(); - try (ClickHouseResponse resp = req.query("select * from test_input_function").execute().get()) { - for (ClickHouseRecord r : resp.records()) { - values.add(new Object[] { r.getValue(0).asObject() }); - } - } - Assert.assertEquals(values.toArray(new Object[0][]), - new Object[][] { new Object[] { "2\t33" }, new Object[] { "3\t333" } }); - } - } - @Test(groups = { "integration" }) public void testLogComment() throws Exception { ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); @@ -175,57 +85,6 @@ public void testLogComment() throws Exception { } } - @Test(groups = "integration") - public void testQueryWithMultipleExternalTables() throws Exception { - ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); - - int tables = 30; - int rows = 10; - try (ClickHouseClient client = ClickHouseClient.builder().build()) { - try (ClickHouseResponse resp = client.connect(server).query("drop table if exists test_ext_data_query") - .execute().get()) { - } - - String ddl = "create table test_ext_data_query (\n" + " Cb String,\n" + " CREATETIME DateTime64(3),\n" - + " TIMESTAMP UInt64,\n" + " Cc String,\n" + " Ca1 UInt64,\n" + " Ca2 UInt64,\n" - + " Ca3 UInt64\n" + ") engine = MergeTree()\n" + "PARTITION BY toYYYYMMDD(CREATETIME)\n" - + "ORDER BY (Cb, CREATETIME, Cc);"; - try (ClickHouseResponse resp = client.connect(server).query(ddl).execute().get()) { - } - } - - String template = "avgIf(Ca1, Cb in L%1$d) as avgCa1%2$d, sumIf(Ca1, Cb in L%1$d) as sumCa1%2$d, minIf(Ca1, Cb in L%1$d) as minCa1%2$d, maxIf(Ca1, Cb in L%1$d) as maxCa1%2$d, anyIf(Ca1, Cb in L%1$d) as anyCa1%2$d, avgIf(Ca2, Cb in L%1$d) as avgCa2%2$d, sumIf(Ca2, Cb in L%1$d) as sumCa2%2$d, minIf(Ca2, Cb in L%1$d) as minCa2%2$d, maxIf(Ca2, Cb in L%1$d) as maxCa2%2$d, anyIf(Ca2, Cb in L%1$d) as anyCa2%2$d, avgIf(Ca3, Cb in L%1$d) as avgCa3%2$d, sumIf(Ca3, Cb in L%1$d) as sumCa3%2$d, minIf(Ca3, Cb in L%1$d) as minCa3%2$d, maxIf(Ca3, Cb in L%1$d) as maxCa3%2$d, anyIf(Ca3, Cb in L%1$d) as anyCa3%2$d"; - StringBuilder sql = new StringBuilder().append("select "); - List extTableList = new ArrayList<>(tables); - for (int i = 0; i < tables; i++) { - sql.append(ClickHouseUtils.format(template, i, i + 1)).append(','); - List valueList = new ArrayList<>(rows); - for (int j = i, size = i + rows; j < size; j++) { - valueList.add(String.valueOf(j)); - } - String dnExtString = String.join("\n", valueList); - InputStream inputStream = new ByteArrayInputStream(dnExtString.getBytes(Charset.forName("UTF-8"))); - ClickHouseExternalTable extTable = ClickHouseExternalTable.builder().name("L" + i).content(inputStream) - .addColumn("Cb", "String").build(); - extTableList.add(extTable); - } - - if (tables > 0) { - sql.deleteCharAt(sql.length() - 1); - } else { - sql.append('*'); - } - sql.append( - " from test_ext_data_query where TIMESTAMP >= 1625796480 and TIMESTAMP < 1625796540 and Cc = 'eth0'"); - - try (ClickHouseClient client = ClickHouseClient.builder().build(); - ClickHouseResponse resp = client.connect(server).query(sql.toString()) - .format(ClickHouseFormat.RowBinaryWithNamesAndTypes).external(extTableList).execute().get()) { - Assert.assertNotNull(resp.getColumns()); - Assert.assertTrue(tables <= 0 || resp.records().iterator().hasNext()); - } - } - @Test(groups = { "integration" }) public void testPost() throws Exception { ClickHouseNode server = getServer(ClickHouseProtocol.HTTP); diff --git a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java index c8b62b8b1..f79bfc001 100644 --- a/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java +++ b/clickhouse-http-client/src/test/java/com/clickhouse/client/http/DefaultHttpConnectionTest.java @@ -1,10 +1,5 @@ package com.clickhouse.client.http; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.HttpURLConnection; -import java.net.URL; - import com.clickhouse.client.BaseIntegrationTest; import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseNode; diff --git a/clickhouse-jdbc/README.md b/clickhouse-jdbc/README.md index 0512cbe0c..b8863bb5a 100644 --- a/clickhouse-jdbc/README.md +++ b/clickhouse-jdbc/README.md @@ -11,7 +11,7 @@ Keep in mind that `clickhouse-jdbc` is synchronous, and in general it has more o com.clickhouse clickhouse-jdbc - 0.3.2-patch3 + 0.3.2-patch4 ``` @@ -29,14 +29,15 @@ Note: `ru.yandex.clickhouse.ClickHouseDriver` and everything under `ru.yandex.cl **Connection Properties**: -| Property | Default | Description | -| -------------------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| continueBatchOnError | `false` | Whether to continue batch processing when error occurred | -| custom_http_headers | | comma separated custom http headers, for example: `User-Agent=client1,X-Gateway-Id=123` | -| custom_http_params | | comma separated custom http query parameters, for example: `extremes=0,max_result_rows=100` | -| jdbcCompliance | `true` | Whether to support standard synchronous UPDATE/DELETE and fake transaction | -| typeMappings | | Customize mapping between ClickHouse data type and Java class, which will affect result of both [getColumnType()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSetMetaData.html#getColumnType-int-) and [getObject(Class)](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-java.lang.String-java.lang.Class-). For example: `UInt128=java.lang.String,UInt256=java.lang.String` | -| wrapperObject | `false` | Whether [getObject()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-int-) should return java.sql.Array / java.sql.Struct for Array / Tuple. | +| Property | Default | Description | +| ------------------------ | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| continueBatchOnError | `false` | Whether to continue batch processing when error occurred | +| createDatabaseIfNotExist | `false` | Whether to create database if it does not exist | +| custom_http_headers | | comma separated custom http headers, for example: `User-Agent=client1,X-Gateway-Id=123` | +| custom_http_params | | comma separated custom http query parameters, for example: `extremes=0,max_result_rows=100` | +| jdbcCompliance | `true` | Whether to support standard synchronous UPDATE/DELETE and fake transaction | +| typeMappings | | Customize mapping between ClickHouse data type and Java class, which will affect result of both [getColumnType()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSetMetaData.html#getColumnType-int-) and [getObject(Class)](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-java.lang.String-java.lang.Class-). For example: `UInt128=java.lang.String,UInt256=java.lang.String` | +| wrapperObject | `false` | Whether [getObject()](https://docs.oracle.com/javase/8/docs/api/java/sql/ResultSet.html#getObject-int-) should return java.sql.Array / java.sql.Struct for Array / Tuple. | Note: please refer to [JDBC specific configuration](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java) and client options([common](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseClientOption.java), [http](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-http-client/src/main/java/com/clickhouse/client/http/config/ClickHouseHttpOption.java) and [grpc](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/config/ClickHouseGrpcOption.java)) for more. @@ -63,6 +64,7 @@ try (Connection conn = dataSource.getConnection("default", "password"); Batch insert... Tips: + 1. Use `PreparedStatement` instead of `Statement` 2. Use [input function](https://clickhouse.com/docs/en/sql-reference/table-functions/input/) whenever possible @@ -170,7 +172,6 @@ try (PreparedStatement stmt = conn.prepareStatement( -
Before 0.3.2... @@ -206,7 +207,7 @@ Additionally, if you have a few instances, you can use `BalancedClickhouseDataSo In order to provide non-JDBC complaint data manipulation functionality, proprietary API exists. Entry point for API is `ClickHouseStatement#write()` method. -1) Importing file into table +1. Importing file into table ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -219,7 +220,7 @@ sth .send(); ``` -2) Configurable send +2. Configurable send ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -233,7 +234,7 @@ sth .send(); ``` -3) Send data in binary formatted with custom user callback +3. Send data in binary formatted with custom user callback ```java import ru.yandex.clickhouse.ClickHouseStatement; @@ -249,4 +250,5 @@ sth.write().send("INSERT INTO test.writer", new ClickHouseStreamCallback() { }, ClickHouseFormat.RowBinary); // RowBinary or Native are supported ``` +
diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java index a7e865c84..9815c6521 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/JdbcConfig.java @@ -21,6 +21,7 @@ public class JdbcConfig { private static final Logger log = LoggerFactory.getLogger(JdbcConfig.class); public static final String PROP_AUTO_COMMIT = "autoCommit"; + public static final String PROP_CREATE_DATABASE = "createDatabaseIfNotExist"; public static final String PROP_CONTINUE_BATCH = "continueBatchOnError"; public static final String PROP_FETCH_SIZE = "fetchSize"; public static final String PROP_JDBC_COMPLIANT = "jdbcCompliant"; @@ -32,6 +33,7 @@ public class JdbcConfig { private static final String BOOLEAN_TRUE = "true"; private static final String DEFAULT_AUTO_COMMIT = BOOLEAN_TRUE; + private static final String DEFAULT_CREATE_DATABASE = BOOLEAN_FALSE; private static final String DEFAULT_CONTINUE_BATCH = BOOLEAN_FALSE; private static final String DEFAULT_FETCH_SIZE = "0"; private static final String DEFAULT_JDBC_COMPLIANT = BOOLEAN_TRUE; @@ -94,6 +96,11 @@ public static List getDriverProperties() { info.description = "Whether to enable auto commit when connection is created."; list.add(info); + info = new DriverPropertyInfo(PROP_CREATE_DATABASE, DEFAULT_CREATE_DATABASE); + info.choices = new String[] { BOOLEAN_TRUE, BOOLEAN_FALSE }; + info.description = "Whether to automatically create database when it does not exist."; + list.add(info); + info = new DriverPropertyInfo(PROP_CONTINUE_BATCH, DEFAULT_CONTINUE_BATCH); info.choices = new String[] { BOOLEAN_TRUE, BOOLEAN_FALSE }; info.description = "Whether to continue batch process when error occurred."; @@ -126,6 +133,7 @@ public static List getDriverProperties() { } private final boolean autoCommit; + private final boolean createDb; private final boolean continueBatch; private final int fetchSize; private final boolean jdbcCompliant; @@ -143,6 +151,7 @@ public JdbcConfig(Properties props) { } this.autoCommit = extractBooleanValue(props, PROP_AUTO_COMMIT, DEFAULT_AUTO_COMMIT); + this.createDb = extractBooleanValue(props, PROP_CREATE_DATABASE, DEFAULT_CREATE_DATABASE); this.continueBatch = extractBooleanValue(props, PROP_CONTINUE_BATCH, DEFAULT_CONTINUE_BATCH); this.fetchSize = extractIntValue(props, PROP_FETCH_SIZE, DEFAULT_FETCH_SIZE); this.jdbcCompliant = extractBooleanValue(props, PROP_JDBC_COMPLIANT, DEFAULT_JDBC_COMPLIANT); @@ -161,6 +170,16 @@ public boolean isAutoCommit() { return autoCommit; } + /** + * Checks whether database should be created automatically when it does not + * exist. + * + * @return true if database should be created automatically; false otherwise + */ + public boolean isCreateDbIfNotExist() { + return createDb; + } + /** * Checks whether batch processing should continue when error occurred. * diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java index 4f62170ac..b75f72357 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/ClickHouseConnectionImpl.java @@ -61,6 +61,48 @@ public class ClickHouseConnectionImpl extends JdbcWrapper implements ClickHouseConnection { private static final Logger log = LoggerFactory.getLogger(ClickHouseConnectionImpl.class); + private static final String CREATE_DB = "create database if not exists `"; + + protected static ClickHouseRecord getServerInfo(ClickHouseNode node, ClickHouseRequest request, + boolean createDbIfNotExist) throws SQLException { + ClickHouseRequest newReq = request.copy(); + if (!createDbIfNotExist) { // in case the database does not exist + newReq.option(ClickHouseClientOption.DATABASE, ""); + } + try (ClickHouseResponse response = newReq.option(ClickHouseClientOption.ASYNC, false) + .option(ClickHouseClientOption.COMPRESS, false).option(ClickHouseClientOption.DECOMPRESS, false) + .option(ClickHouseClientOption.FORMAT, ClickHouseFormat.RowBinaryWithNamesAndTypes) + .query("select currentUser(), timezone(), version() FORMAT RowBinaryWithNamesAndTypes") + .execute().get()) { + return response.firstRecord(); + } catch (InterruptedException | CancellationException e) { + // not going to happen as it's synchronous call + Thread.currentThread().interrupt(); + throw SqlExceptionUtils.forCancellation(e); + } catch (Exception e) { + SQLException sqlExp = SqlExceptionUtils.handle(e); + if (createDbIfNotExist && sqlExp.getErrorCode() == 81) { + String db = node.getDatabase(request.getConfig()); + try (ClickHouseResponse resp = newReq.use("") + .query(new StringBuilder(CREATE_DB.length() + 1 + db.length()).append(CREATE_DB).append(db) + .append('`').toString()) + .execute().get()) { + return getServerInfo(node, request, false); + } catch (InterruptedException | CancellationException ex) { + // not going to happen as it's synchronous call + Thread.currentThread().interrupt(); + throw SqlExceptionUtils.forCancellation(ex); + } catch (SQLException ex) { + throw ex; + } catch (Exception ex) { + throw SqlExceptionUtils.handle(ex); + } + } else { + throw sqlExp; + } + } + } + private final JdbcConfig jdbcConf; private final ClickHouseClient client; @@ -185,51 +227,40 @@ public ClickHouseConnectionImpl(ConnectionInfo connInfo) throws SQLException { .nodeSelector(ClickHouseNodeSelector.of(node.getProtocol())).build(); clientRequest = client.connect(node); ClickHouseConfig config = clientRequest.getConfig(); - String currentDb = null; String currentUser = null; TimeZone timeZone = null; ClickHouseVersion version = null; if (config.hasServerInfo()) { // when both serverTimeZone and serverVersion are configured timeZone = config.getServerTimeZone(); version = config.getServerVersion(); + if (jdbcConf.isCreateDbIfNotExist()) { + getServerInfo(node, clientRequest, true); + } } else { - try (ClickHouseResponse response = clientRequest.copy().option(ClickHouseClientOption.ASYNC, false) - .option(ClickHouseClientOption.COMPRESS, false).option(ClickHouseClientOption.DECOMPRESS, false) - .option(ClickHouseClientOption.FORMAT, ClickHouseFormat.RowBinaryWithNamesAndTypes) - .query("select currentDatabase(), currentUser(), timezone(), version() FORMAT RowBinaryWithNamesAndTypes") - .execute().get()) { - ClickHouseRecord r = response.firstRecord(); - currentDb = r.getValue(0).asString(); - currentUser = r.getValue(1).asString(); - String tz = r.getValue(2).asString(); - String ver = r.getValue(3).asString(); - version = ClickHouseVersion.of(ver); - // https://github.com/ClickHouse/ClickHouse/commit/486d63864bcc6e15695cd3e9f9a3f83a84ec4009 - if (version.check("(,20.7)")) { - throw SqlExceptionUtils - .unsupportedError("Sorry this driver only supports ClickHouse server 20.7 or above"); - } - if (ClickHouseChecker.isNullOrBlank(tz)) { - tz = "UTC"; - } - // tsTimeZone.hasSameRules(ClickHouseValues.UTC_TIMEZONE) - timeZone = "UTC".equals(tz) ? ClickHouseValues.UTC_TIMEZONE : TimeZone.getTimeZone(tz); - - // update request and corresponding config - clientRequest.option(ClickHouseClientOption.SERVER_TIME_ZONE, tz) - .option(ClickHouseClientOption.SERVER_VERSION, ver); - } catch (InterruptedException | CancellationException e) { - // not going to happen as it's synchronous call - Thread.currentThread().interrupt(); - throw SqlExceptionUtils.forCancellation(e); - } catch (Exception e) { - throw SqlExceptionUtils.handle(e); + ClickHouseRecord r = getServerInfo(node, clientRequest, jdbcConf.isCreateDbIfNotExist()); + currentUser = r.getValue(0).asString(); + String tz = r.getValue(1).asString(); + String ver = r.getValue(2).asString(); + version = ClickHouseVersion.of(ver); + // https://github.com/ClickHouse/ClickHouse/commit/486d63864bcc6e15695cd3e9f9a3f83a84ec4009 + if (version.check("(,20.7)")) { + throw SqlExceptionUtils + .unsupportedError("Sorry this driver only supports ClickHouse server 20.7 or above"); + } + if (ClickHouseChecker.isNullOrBlank(tz)) { + tz = "UTC"; } + // tsTimeZone.hasSameRules(ClickHouseValues.UTC_TIMEZONE) + timeZone = "UTC".equals(tz) ? ClickHouseValues.UTC_TIMEZONE : TimeZone.getTimeZone(tz); + + // update request and corresponding config + clientRequest.option(ClickHouseClientOption.SERVER_TIME_ZONE, tz) + .option(ClickHouseClientOption.SERVER_VERSION, ver); } this.autoCommit = true; this.closed = false; - this.database = currentDb != null ? currentDb : config.getDatabase(); + this.database = config.getDatabase(); this.clientRequest.use(this.database); this.readOnly = false; this.networkTimeout = 0; diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java index ceb8d77e8..9fb3ba02c 100644 --- a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/internal/InputBasedPreparedStatement.java @@ -270,7 +270,8 @@ public int[] executeBatch() throws SQLException { clearBatch(); } - Arrays.fill(results, result); + // FIXME grpc and tcp by default can provides accurate result + Arrays.fill(results, 1); return results; } diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java index 653d6c1ee..96bcf7da6 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseConnectionImpl.java @@ -20,6 +20,7 @@ import java.sql.Savepoint; import java.sql.Statement; import java.sql.Struct; +import java.util.Collections; import java.util.Map; import java.util.Properties; import java.util.TimeZone; @@ -36,6 +37,7 @@ import ru.yandex.clickhouse.except.ClickHouseUnknownException; import ru.yandex.clickhouse.settings.ClickHouseConnectionSettings; import ru.yandex.clickhouse.settings.ClickHouseProperties; +import ru.yandex.clickhouse.settings.ClickHouseQueryParam; import ru.yandex.clickhouse.util.ClickHouseHttpClientBuilder; import ru.yandex.clickhouse.util.LogProxy; import ru.yandex.clickhouse.util.Utils; @@ -92,7 +94,9 @@ private void initConnection(ClickHouseProperties properties) throws SQLException timezone = serverTimeZone; serverVersion = ""; - try (Statement s = createStatement(); ResultSet rs = s.executeQuery("select timezone(), version()")) { + try (ClickHouseStatement s = createStatement(); + ResultSet rs = s.executeQuery("select timezone(), version()", + Collections.singletonMap(ClickHouseQueryParam.DATABASE, ""))) { if (rs.next()) { serverTimeZone = TimeZone.getTimeZone(rs.getString(1)); serverVersion = rs.getString(2); diff --git a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java index 6a22d2c81..0c355b27e 100644 --- a/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java +++ b/clickhouse-jdbc/src/main/java/ru/yandex/clickhouse/ClickHouseStatementImpl.java @@ -851,7 +851,8 @@ private List getUrlQueryParams(String sql, List params = properties.buildQueryParams(true); - if (!ignoreDatabase) { + String db = params.getOrDefault(ClickHouseQueryParam.DATABASE, currentDatabase); + if (!ignoreDatabase && !Utils.isNullOrEmptyString(db)) { params.put(ClickHouseQueryParam.DATABASE, currentDatabase); } diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java index f60ed84a1..d95bf15a3 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseConnectionTest.java @@ -1,13 +1,24 @@ package com.clickhouse.jdbc; import java.sql.Array; +import java.sql.ResultSet; import java.sql.SQLException; +import java.sql.Statement; import java.util.Properties; +import java.util.UUID; + +import com.clickhouse.client.ClickHouseUtils; +import com.clickhouse.client.config.ClickHouseClientOption; import org.testng.Assert; import org.testng.annotations.Test; public class ClickHouseConnectionTest extends JdbcIntegrationTest { + @Override + public ClickHouseConnection newConnection(Properties properties) throws SQLException { + return newDataSource(properties).getConnection(); + } + @Test(groups = "integration") public void testCreateArray() throws SQLException { try (ClickHouseConnection conn = newConnection(new Properties())) { @@ -15,4 +26,47 @@ public void testCreateArray() throws SQLException { Assert.assertEquals(array.getArray(), new byte[0]); } } + + @Test(groups = "integration") + public void testNonExistDatabase() throws Exception { + String database = UUID.randomUUID().toString(); + Properties props = new Properties(); + props.setProperty(ClickHouseClientOption.DATABASE.getKey(), database); + SQLException exp = null; + try (ClickHouseConnection conn = newConnection(props)) { + // do nothing + } + Assert.assertNull(exp, "Should not have SQLException even the database does not exist"); + + try (ClickHouseConnection conn = newConnection(props)) { + Assert.assertEquals(conn.getSchema(), database); + try (Statement stmt = conn.createStatement(); ResultSet rs = stmt.executeQuery("select 1")) { + Assert.fail("Should have SQLException"); + } + } catch (SQLException e) { + exp = e; + } + Assert.assertNotNull(exp, "Should have SQLException since the database does not exist"); + Assert.assertEquals(exp.getErrorCode(), 81); + + props.setProperty(JdbcConfig.PROP_CREATE_DATABASE, Boolean.TRUE.toString()); + try (ClickHouseConnection conn = newConnection(props)) { + exp = null; + } + Assert.assertNull(exp, "Should not have SQLException because database will be created automatically"); + + props.setProperty(JdbcConfig.PROP_CREATE_DATABASE, Boolean.FALSE.toString()); + try (ClickHouseConnection conn = newConnection(props)) { + Assert.assertEquals(conn.getSchema(), database); + try (Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery( + ClickHouseUtils.format("select * from system.databases where name='%s'", database))) { + Assert.assertTrue(rs.next(), "Should have at least one record in system.databases table"); + Assert.assertEquals(rs.getString("name"), database); + Assert.assertFalse(rs.next(), "Should have only one record in system.databases table"); + exp = new SQLException(); + } + } + Assert.assertNotNull(exp, "Should not have SQLException because the database has been created"); + } } diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java index a8fd937e4..5221226d2 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHousePreparedStatementTest.java @@ -18,6 +18,7 @@ import java.util.UUID; import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.data.ClickHouseBitmap; import com.clickhouse.client.data.ClickHouseExternalTable; @@ -484,6 +485,11 @@ public void testBatchQuery() throws SQLException { @Test(groups = "integration") public void testQueryWithExternalTable() throws SQLException { + // FIXME grpc seems has problem dealing with session + if (DEFAULT_PROTOCOL == ClickHouseProtocol.GRPC) { + return; + } + try (ClickHouseConnection conn = newConnection(new Properties()); PreparedStatement stmt = conn.prepareStatement( "SELECT bitmapContains(my_bitmap, toUInt32(1)) as v1, bitmapContains(my_bitmap, toUInt32(2)) as v2 from {tt 'ext_table'}")) { @@ -518,6 +524,29 @@ public void testArrayParameter(String t, Object v) throws SQLException { } } + @Test(groups = "integration") + public void testInsertStringAsArray() throws Exception { + try (ClickHouseConnection conn = newConnection(new Properties()); + Statement s = conn.createStatement(); + PreparedStatement stmt = conn.prepareStatement( + "insert into test_array_insert(id, a, b) values (?,?,?)")) { + s.execute("drop table if exists test_array_insert;" + + "create table test_array_insert(id UInt32, a Array(Int16), b Array(Nullable(UInt32)))engine=Memory"); + + stmt.setString(1, "1"); + stmt.setString(2, "[1,2,3]"); + stmt.setString(3, "[3,null,1]"); + Assert.assertEquals(stmt.executeUpdate(), 1); + + ResultSet rs = s.executeQuery("select * from test_array_insert order by id"); + Assert.assertTrue(rs.next()); + Assert.assertEquals(rs.getInt(1), 1); + Assert.assertEquals(rs.getObject(2), new short[] { 1, 2, 3 }); + Assert.assertEquals(rs.getObject(3), new long[] { 3, 0, 1 }); + Assert.assertFalse(rs.next()); + } + } + @Test(groups = "integration") public void testInsertWithFunction() throws Exception { try (ClickHouseConnection conn = newConnection(new Properties()); diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java index 677107ce8..4aac80e4a 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/ClickHouseStatementTest.java @@ -31,6 +31,7 @@ import com.clickhouse.client.ClickHouseClient; import com.clickhouse.client.ClickHouseDataType; import com.clickhouse.client.ClickHouseParameterizedQuery; +import com.clickhouse.client.ClickHouseProtocol; import com.clickhouse.client.ClickHouseValues; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.data.ClickHouseDateTimeValue; @@ -122,6 +123,10 @@ public void testMutation() throws SQLException { @Test(groups = "integration") public void testAsyncInsert() throws SQLException { + if (DEFAULT_PROTOCOL != ClickHouseProtocol.HTTP) { + return; + } + Properties props = new Properties(); try (ClickHouseConnection conn = newConnection(props)) { if (conn.getServerVersion().check("(,21.12)")) { diff --git a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java index 854c2a691..700e95fc9 100644 --- a/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java +++ b/clickhouse-jdbc/src/test/java/com/clickhouse/jdbc/JdbcIntegrationTest.java @@ -15,6 +15,9 @@ public abstract class JdbcIntegrationTest extends BaseIntegrationTest { private static final String CLASS_PREFIX = "ClickHouse"; private static final String CLASS_SUFFIX = "Test"; + protected static final ClickHouseProtocol DEFAULT_PROTOCOL = ClickHouseProtocol + .valueOf(System.getProperty("protocol", "http").toUpperCase()); + protected final String dbName; protected String buildJdbcUrl(ClickHouseProtocol protocol, String prefix, String url) { @@ -22,9 +25,13 @@ protected String buildJdbcUrl(ClickHouseProtocol protocol, String prefix, String return url; } + if (protocol == null) { + protocol = DEFAULT_PROTOCOL; + } + StringBuilder builder = new StringBuilder(); if (prefix == null || prefix.isEmpty()) { - builder.append("jdbc:clickhouse://"); + builder.append("jdbc:clickhouse:").append(protocol.name().toLowerCase()).append("://"); } else if (!prefix.startsWith("jdbc:")) { builder.append("jdbc:").append(prefix); } else { @@ -87,7 +94,7 @@ public ClickHouseDataSource newDataSource(String url) throws SQLException { } public ClickHouseDataSource newDataSource(String url, Properties properties) throws SQLException { - return new ClickHouseDataSource(buildJdbcUrl(ClickHouseProtocol.HTTP, null, url), properties); + return new ClickHouseDataSource(buildJdbcUrl(DEFAULT_PROTOCOL, null, url), properties); } public ClickHouseConnection newConnection() throws SQLException { diff --git a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java index 1b3cc0ca1..febcf3413 100644 --- a/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java +++ b/clickhouse-jdbc/src/test/java/ru/yandex/clickhouse/integration/ClickHouseConnectionImplTest.java @@ -4,11 +4,13 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.Statement; +import java.util.UUID; import javax.sql.DataSource; import org.testng.annotations.Test; +import ru.yandex.clickhouse.ClickHouseConnection; import ru.yandex.clickhouse.ClickHouseDataSource; import ru.yandex.clickhouse.JdbcIntegrationTest; import ru.yandex.clickhouse.except.ClickHouseException; @@ -139,6 +141,20 @@ public void testDefaultDatabase() throws Exception { } } + @Test(groups = "integration") + public void testNonExistDatabase() throws Exception { + String database = UUID.randomUUID().toString(); + ClickHouseProperties props = new ClickHouseProperties(); + props.setUser("default"); + props.setPassword(""); + props.setDatabase(database); + ClickHouseDataSource ds = newDataSource(props); + try (ClickHouseConnection conn = ds.getConnection()) { + conn.createStatement().execute("create database `" + database + "`; " + + "drop database `" + database + "`"); + } + } + private static void assertSuccess(DataSource dataSource) throws Exception { Connection connection = dataSource.getConnection(); assertTrue(connection.createStatement().execute("SELECT 1")); diff --git a/clickhouse-jdbc/src/test/resources/simplelogger.properties b/clickhouse-jdbc/src/test/resources/simplelogger.properties index 7589a0e57..d2675fbea 100644 --- a/clickhouse-jdbc/src/test/resources/simplelogger.properties +++ b/clickhouse-jdbc/src/test/resources/simplelogger.properties @@ -1,5 +1,5 @@ org.slf4j.simpleLogger.defaultLogLevel=info -org.slf4j.simpleLogger.log.com.clickhouse.client=debug +org.slf4j.simpleLogger.log.com.clickhouse.jdbc=debug org.slf4j.simpleLogger.showDateTime=true org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z org.slf4j.simpleLogger.showThreadName=true diff --git a/examples/grpc/pom.xml b/examples/grpc/pom.xml index 9f012ed79..1993f057c 100644 --- a/examples/grpc/pom.xml +++ b/examples/grpc/pom.xml @@ -56,7 +56,7 @@ UTF-8 UTF-8 - 0.3.2-patch3 + 0.3.2-patch4 3.8.1 diff --git a/examples/jdbc/pom.xml b/examples/jdbc/pom.xml index 664f21344..8d8762766 100644 --- a/examples/jdbc/pom.xml +++ b/examples/jdbc/pom.xml @@ -56,7 +56,7 @@ UTF-8 UTF-8 - 0.3.2-patch3 + 0.3.2-patch4 3.8.1 diff --git a/pom.xml b/pom.xml index 9b075830d..6aa3bda32 100644 --- a/pom.xml +++ b/pom.xml @@ -92,12 +92,12 @@ 2.0.0-alpha5 3.12.4 2.32.0 - 1.16.2 + 1.16.3 7.4.0 - 2.7.4 - 8.0.27 - 42.3.1 + 3.0.3 + 8.0.28 + 42.3.2 1.1.1