diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5c01b8515..b89a39843 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,7 @@ concurrency: cancel-in-progress: true jobs: - compile: + compile8: runs-on: ubuntu-latest timeout-minutes: 10 name: Compile using JDK 8 @@ -56,10 +56,53 @@ jobs: run: mvn --batch-mode --show-version --strict-checksums --threads C1 -Dmaven.wagon.rto=30000 -Dj8 -DskipITs install - name: Compile examples run: for d in $(ls -d `pwd`/examples/*/); do cd $d && mvn clean compile; done + + compile: + runs-on: ubuntu-latest + timeout-minutes: 10 + name: Compile using JDK 17 + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Check out PR + run: | + git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ + origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr + if: github.event.inputs.pr != '' + - name: Install JDK 17 and Maven + uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: | + 8 + 17 + cache: "maven" + - name: Setup Toolchain + shell: bash + run: | + mkdir -p $HOME/.m2 \ + && cat << EOF > $HOME/.m2/toolchains.xml + + + + jdk + + 17 + + + ${{ env.JAVA_HOME }} + + + + EOF + - name: Build and install libraries + run: mvn --batch-mode --show-version --strict-checksums --threads C1 -Dmaven.wagon.rto=30000 -DskipITs install + - name: Compile examples + run: for d in $(ls -d `pwd`/examples/*/); do cd $d && mvn clean compile; done test-cli-client: runs-on: ubuntu-latest - needs: compile + needs: compile8 timeout-minutes: 10 name: CLI client + CH 22.8 steps: @@ -114,7 +157,7 @@ jobs: git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr if: github.event.inputs.pr != '' - - name: Install JDK 8 and Maven + - name: Install JDK 17 and Maven uses: actions/setup-java@v3 with: distribution: "temurin" @@ -174,7 +217,7 @@ jobs: git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr if: github.event.inputs.pr != '' - - name: Install JDK 8 and Maven + - name: Install JDK 17 and Maven uses: actions/setup-java@v3 with: distribution: "temurin" @@ -234,7 +277,7 @@ jobs: git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \ origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr if: github.event.inputs.pr != '' - - name: Install JDK 8 and Maven + - name: Install JDK 17 and Maven uses: actions/setup-java@v3 with: distribution: "temurin" @@ -277,7 +320,7 @@ jobs: test-timezone-support: runs-on: ubuntu-latest - needs: compile + needs: compile8 strategy: matrix: serverTz: @@ -312,7 +355,7 @@ jobs: with: distribution: "temurin" java-version: 8 - cache: "maven" + cache: "" - name: Install Java client run: mvn --also-make --batch-mode --projects clickhouse-cli-client,clickhouse-grpc-client,clickhouse-http-client -Dj8 -DskipTests install - name: Test JDBC and R2DBC drivers diff --git a/.gitignore b/.gitignore index e0c1be403..6ca6b5e30 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .DS_Store # Java Files +*.bgv *.class *.jar *.war diff --git a/README.md b/README.md index e87e5cb09..07ccfd0af 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Java libraries for connecting to ClickHouse and processing data in various forma | ----------------- | ----------------------------------------------------------------------------------- | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | API | [JDBC](https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/) | :white_check_mark: | | | | [R2DBC](https://r2dbc.io/) | :white_check_mark: | supported since 0.4.0 | +| Query Language | SQL | :white_check_mark: | | +| | [PRQL](https://prql-lang.org/) | :x: | | +| | [GraphQL](https://graphql.org/) | :x: | | | Protocol | [HTTP](https://clickhouse.com/docs/en/interfaces/http/) | :white_check_mark: | recommended, defaults to `java.net.HttpURLConnection` and it can be changed to `java.net.http.HttpClient`(unstable) or `Apache HTTP Client 5`. Note that the latter was added in 0.4.0 to support custom socket options. | | | [gRPC](https://clickhouse.com/docs/en/interfaces/grpc/) | :white_check_mark: | still experimental, works with 22.3+, known to has [issue](https://github.com/ClickHouse/ClickHouse/issues/28671#issuecomment-1087049993) when using LZ4 compression | | | [TCP/Native](https://clickhouse.com/docs/en/interfaces/tcp/) | :white_check_mark: | `clickhouse-cli-client`(wrapper of ClickHouse native command-line client) was added in 0.3.2-patch10, `clickhouse-tcp-client` will be available in 0.5 | @@ -22,7 +25,7 @@ Java libraries for connecting to ClickHouse and processing data in various forma | | [zstd](https://facebook.github.io/zstd/) | :white_check_mark: | supported since 0.4.0, works with ClickHouse 22.10+ | | Data Format | RowBinary | :white_check_mark: | `RowBinaryWithNamesAndTypes` for query and `RowBinary` for insertion | | | TabSeparated | :white_check_mark: | Does not support as many data types as RowBinary | -| Data Type | AggregatedFunction | :x: | limited to `groupBitmap`, and known to have issue with 64bit bitmap | +| Data Type | AggregatedFunction | :x: | :warning: limited to `groupBitmap`; 64bit bitmap was NOT working properly before 0.4.1 | | | Array(\*) | :white_check_mark: | | | | Bool | :white_check_mark: | | | | Date\* | :white_check_mark: | | @@ -33,10 +36,10 @@ Java libraries for connecting to ClickHouse and processing data in various forma | | Int\*, UInt\* | :white_check_mark: | UInt64 is mapped to `long` | | | IPv\* | :white_check_mark: | | | | Map(\*) | :white_check_mark: | | -| | Nested(\*) | :white_check_mark: | | +| | Nested(\*) | :white_check_mark: | :warning: broken before 0.4.1 | | | Object('JSON') | :white_check_mark: | supported since 0.3.2-patch8 | | | SimpleAggregateFunction | :white_check_mark: | | -| | \*String | :white_check_mark: | | +| | \*String | :white_check_mark: | :warning: requires `use_binary_string=true` for binary string support since v0.4.0 | | | Tuple(\*) | :white_check_mark: | | | | UUID | :white_check_mark: | | | High Availability | Load Balancing | :white_check_mark: | supported since 0.3.2-patch10 | diff --git a/clickhouse-cli-client/pom.xml b/clickhouse-cli-client/pom.xml index 5f7190247..253ef81dd 100644 --- a/clickhouse-cli-client/pom.xml +++ b/clickhouse-cli-client/pom.xml @@ -1,4 +1,6 @@ - + 4.0.0 @@ -22,7 +24,8 @@ org.lz4 - lz4-java + lz4-pure-java + true @@ -87,6 +90,13 @@ true true true + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + org.lz4:lz4-pure-java + + net.jpountz @@ -94,10 +104,8 @@ - - - - + com.clickhouse.client.cli @@ -107,13 +115,7 @@ *:* - **/darwin/** - **/linux/** - **/win32/** **/module-info.class - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native-image/** diff --git a/clickhouse-client/pom.xml b/clickhouse-client/pom.xml index d2bf525e3..0aee47edb 100644 --- a/clickhouse-client/pom.xml +++ b/clickhouse-client/pom.xml @@ -34,6 +34,18 @@ + + com.graphql-java + graphql-java + ${graphql.version} + true + + + org.slf4j + slf4j-api + + + com.google.code.gson gson diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java index 8e6623649..c365a3942 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseClient.java @@ -18,6 +18,7 @@ import java.util.concurrent.TimeoutException; import java.util.function.Function; +import com.clickhouse.client.ClickHouseRequest.Mutation; import com.clickhouse.client.config.ClickHouseClientOption; import com.clickhouse.client.config.ClickHouseDefaults; import com.clickhouse.config.ClickHouseBufferingMode; @@ -891,12 +892,14 @@ default boolean accept(ClickHouseProtocol protocol) { * Connects to one or more ClickHouse servers. Same as * {@code connect(ClickHouseNodes.of(uri))}. * - * @param enpoints non-empty URIs separated by comman + * @param endpoints non-empty URIs separated by comma * @return non-null request object holding references to this client and node * provider + * @deprecated will be dropped in 0.5, please use {@link #read(String)} instead */ - default ClickHouseRequest connect(String enpoints) { - return connect(ClickHouseNodes.of(enpoints)); + @Deprecated + default ClickHouseRequest connect(String endpoints) { + return read(endpoints); } /** @@ -905,24 +908,73 @@ default ClickHouseRequest connect(String enpoints) { * @param nodes non-null list of servers to connect to * @return non-null request object holding references to this client and node * provider + * @deprecated will be dropped in 0.5, please use {@link #read(ClickHouseNodes)} + * instead */ + @Deprecated default ClickHouseRequest connect(ClickHouseNodes nodes) { - return new ClickHouseRequest<>(this, ClickHouseChecker.nonNull(nodes, "Nodes"), - null, nodes.template.config.getAllOptions(), false); + return read(nodes); } /** * Connects to a ClickHouse server. * - * @param node non-null server to connect to + * @param node non-null server for read * @return non-null request object holding references to this client and node * provider + * @deprecated will be dropped in 0.5, please use {@link #read(ClickHouseNode)} + * instead */ + @Deprecated default ClickHouseRequest connect(ClickHouseNode node) { + return read(node); + } + + /** + * Reads from one or more ClickHouse servers. Same as + * {@code read(ClickHouseNodes.of(uri))}. + * + * @param endpoints non-empty connection string separated by comma + * @return non-null request object holding references to this client and node + * provider + */ + default ClickHouseRequest read(String endpoints) { + return read(ClickHouseNodes.of(endpoints)); + } + + /** + * Reads from a list of managed ClickHouse servers. + * + * @param nodes non-null list of servers for read + * @return non-null request object holding references to this client and node + * provider + */ + default ClickHouseRequest read(ClickHouseNodes nodes) { + return new ClickHouseRequest<>(this, ClickHouseChecker.nonNull(nodes, "Nodes"), + null, nodes.template.config.getAllOptions(), false); + } + + /** + * Reads from a ClickHouse server. + * + * @param node non-null server for read + * @return non-null request object holding references to this client and server + */ + default ClickHouseRequest read(ClickHouseNode node) { return new ClickHouseRequest<>(this, ClickHouseChecker.nonNull(node, "Node"), null, node.config.getAllOptions(), false); } + /** + * Writes into a ClickHouse server. + * + * @param server non-null server for write + * @return non-null request object holding references to this client and server + */ + default Mutation write(ClickHouseNode server) { + return read(server).write(); + } + /** * Connects to a ClickHouse server defined by the given * {@link java.util.function.Function}. You can pass either diff --git a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java index d4ef84abe..263cceeee 100644 --- a/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java +++ b/clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseNode.java @@ -1016,6 +1016,15 @@ public String getBaseUri() { return this.baseUri; } + /** + * Gets configuration. + * + * @return non-null configuration + */ + public ClickHouseConfig getConfig() { + return this.config; + } + /** * Gets credentials for accessing this node. Use * {@link ClickHouseConfig#getDefaultCredentials()} if this is not present. diff --git a/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/native-image.properties b/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/native-image.properties new file mode 100755 index 000000000..758eb38cc --- /dev/null +++ b/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/native-image.properties @@ -0,0 +1 @@ +Args = -H:IncludeResources=${.}/resource-config.json diff --git a/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/resource-config.json b/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/resource-config.json new file mode 100644 index 000000000..7ffc5fa57 --- /dev/null +++ b/clickhouse-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-client/resource-config.json @@ -0,0 +1,13 @@ +{ + "resources": { + "includes": [ + { + "pattern": "\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E" + }, + { + "pattern": "\\QMETA-INF/services/com.clickhouse.client.ClickHouseSslContextProvider\\E" + } + ] + }, + "bundles": [] +} diff --git a/clickhouse-data/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-data/native-image.properties b/clickhouse-data/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-data/native-image.properties new file mode 100755 index 000000000..c898d0911 --- /dev/null +++ b/clickhouse-data/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-data/native-image.properties @@ -0,0 +1 @@ +Args = --initialize-at-build-time=com.clickhouse.data.ClickHouseFormat diff --git a/clickhouse-grpc-client/pom.xml b/clickhouse-grpc-client/pom.xml index 8041098ff..c1cde0163 100644 --- a/clickhouse-grpc-client/pom.xml +++ b/clickhouse-grpc-client/pom.xml @@ -32,10 +32,6 @@ - - org.lz4 - lz4-java - com.aayushatharva.brotli4j @@ -70,15 +66,35 @@ com.google.code.gson gson + true org.apache.commons commons-compress + true io.grpc grpc-protobuf true + + + com.google.errorprone + error_prone_annotations + + + com.google.code.findbugs + jsr305 + + + org.checkerframework + checker-qual + + + com.google.j2objc + j2objc-annotations + + org.tukaani @@ -150,6 +166,16 @@ true true true + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + com.clickhouse:io.grpc + io.grpc:grpc-netty-shaded + io.grpc:grpc-okhttp + org.apache.commons:commons-compress + + com.google @@ -167,10 +193,6 @@ io.perfmark ${shade.base}.perfmark - - net.jpountz - ${shade.base}.jpountz - okio ${shade.base}.okio @@ -195,230 +217,13 @@ - - com.aayushatharva.brotli4j:* - - ** - - - - com.github.luben:zstd-jni - - ** - - - - org.tukaani:xz - - ** - - - - org.xerial.snappy:snappy-java - - ** - - - - *:* - - google/** - javax/** - mozilla/** - org/checkerframework/** - org/codehaus/** - **/darwin/** - **/linux/** - **/win32/** - **/module-info.class - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - - - - - - - shade-netty - package - - shade - - - true - true - true - true - netty - - - com.google - ${shade.base}.google - - - io.grpc - ${shade.base}.grpc - - - io.opencensus - ${shade.base}.opencensus - - - io.perfmark - ${shade.base}.perfmark - - - net.jpountz - ${shade.base}.jpountz - - - org.apache - ${shade.base}.apache - - - - - - - - - - com.aayushatharva.brotli4j:* - - ** - - - - com.github.luben:zstd-jni - - ** - - - - org.tukaani:xz - - ** - - - - org.xerial.snappy:snappy-java - - ** - - - - *:* - - google/** - io/grpc/okhttp/** - javax/** - mozilla/** - okio/** - org/checkerframework/** - org/codehaus/** - **/module-info.class - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native-image/** - - - - - - - shade-okhttp - package - - shade - - - true - true - true - true - okhttp - - - com.google - ${shade.base}.google - - - io.grpc - ${shade.base}.grpc - - - io.perfmark - ${shade.base}.perfmark - - - io.opencensus - ${shade.base}.opencensus - - - net.jpountz - ${shade.base}.jpountz - - - okio - ${shade.base}.okio - - - org.apache - ${shade.base}.apache - - - - - - - - - - com.aayushatharva.brotli4j:* - - ** - - - - com.github.luben:zstd-jni - - ** - - - - org.tukaani:xz - - ** - - - - org.xerial.snappy:snappy-java - - ** - - *:* google/** - io/grpc/netty/** - javax/** - mozilla/** org/checkerframework/** org/codehaus/** **/module-info.class - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** diff --git a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java index c58b84b1d..da3434063 100644 --- a/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java +++ b/clickhouse-grpc-client/src/main/java/com/clickhouse/client/grpc/ClickHouseGrpcChannelFactory.java @@ -79,7 +79,17 @@ public ProxiedSocketAddress proxyFor(SocketAddress arg0) throws IOException { } public static ClickHouseGrpcChannelFactory getFactory(ClickHouseConfig config, ClickHouseNode server) { - return (config.getBoolOption(ClickHouseGrpcOption.USE_OKHTTP)) + if (!config.hasOption(ClickHouseGrpcOption.USE_OKHTTP)) { // default + ClickHouseGrpcChannelFactory factory = null; + try { + factory = new NettyChannelFactoryImpl(config, server); + } catch (NoClassDefFoundError e) { + factory = new OkHttpChannelFactoryImpl(config, server); + } + return factory; + } + + return config.getBoolOption(ClickHouseGrpcOption.USE_OKHTTP) ? new OkHttpChannelFactoryImpl(config, server) : new NettyChannelFactoryImpl(config, server); } diff --git a/clickhouse-http-client/pom.xml b/clickhouse-http-client/pom.xml index 4adb29901..247654593 100644 --- a/clickhouse-http-client/pom.xml +++ b/clickhouse-http-client/pom.xml @@ -40,13 +40,22 @@ org.apache.httpcomponents.client5 httpclient5 - true + + org.apache.httpcomponents.core5 + httpcore5 + org.slf4j slf4j-api + true + + + org.apache.httpcomponents.core5 + httpcore5 + true org.brotli @@ -55,7 +64,7 @@ org.lz4 - lz4-java + lz4-pure-java true @@ -117,6 +126,13 @@ true true true + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + org.lz4:lz4-pure-java + + net.jpountz @@ -124,12 +140,6 @@ - - - @@ -138,48 +148,10 @@ - - com.github.luben:zstd-jni - - ** - - - - org.apache.commons:commons-compress - - ** - - - - org.apache.httpcomponents.client5:httpclient5 - - ** - - - - org.brotli:dec - - ** - - - - org.tukaani:xz - - ** - - *:* - mozilla/** - org/apache/** - **/darwin/** - **/linux/** - **/win32/** **/module-info.class - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native-image/** diff --git a/clickhouse-http-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-http-client/native-image.properties b/clickhouse-http-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-http-client/native-image.properties new file mode 100755 index 000000000..313e3bea4 --- /dev/null +++ b/clickhouse-http-client/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-http-client/native-image.properties @@ -0,0 +1 @@ +Args = --enable-url-protocols=http,https diff --git a/clickhouse-jdbc/pom.xml b/clickhouse-jdbc/pom.xml index 534407746..997dadb3c 100644 --- a/clickhouse-jdbc/pom.xml +++ b/clickhouse-jdbc/pom.xml @@ -20,6 +20,9 @@ 4.1.4 JDBC 4.2 + + clickhouse-jdbc-bin + com.clickhouse.jdbc.Main @@ -27,16 +30,31 @@ ${project.parent.groupId} clickhouse-cli-client ${revision} + true + + + com.clickhouse + clickhouse-client + + ${project.parent.groupId} clickhouse-grpc-client ${revision} + true + + + com.clickhouse + clickhouse-client + + ${project.parent.groupId} clickhouse-http-client ${revision} + true ${project.parent.groupId} @@ -49,20 +67,40 @@ - + + org.apache.commons + commons-compress + true + org.apache.httpcomponents.client5 httpclient5 + + org.apache.httpcomponents.core5 + httpcore5 + org.slf4j slf4j-api + true + + + org.apache.httpcomponents.core5 + httpcore5 + true org.lz4 lz4-java + true + + + org.lz4 + lz4-pure-java + true @@ -94,6 +132,91 @@ + + + + native + + + + org.graalvm.buildtools + native-maven-plugin + true + + + build-native + + compile-no-fork + + package + + + test-native + + test + + test + + + + ${imageName} + ${mainClass} + + --no-fallback + + + + ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar + + + true + + + + + org.codehaus.mojo + exec-maven-plugin + + + java-agent + + exec + + + java + ${project.build.directory} + + -classpath + + + ${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar + + ${mainClass} + + + + + native + + exec + + + ${project.build.directory}/${imageName} + ${project.build.directory} + + + + + + + + + @@ -130,6 +253,7 @@ maven-jar-plugin + false ${spec.title} ${spec.version} @@ -145,6 +269,7 @@ jar-with-dependencies + false true true @@ -157,9 +282,8 @@ maven-shade-plugin + specific protocol; '-shaded' is a combination of '-cli' and '-http' with Apache http client; + '-all' is fat for a reason as it includes everything we have ;) --> shade package @@ -172,6 +296,17 @@ true true shaded + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + com.clickhouse:clickhouse-cli-client + com.clickhouse:clickhouse-http-client + org.apache.httpcomponents.client5:httpclient5 + org.apache.httpcomponents.core5:httpcore5 + org.lz4:lz4-pure-java + + org.apache @@ -193,47 +328,17 @@ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> ${project.groupId}.jdbc + ${mainClass} ${spec.title} ${spec.version} - - ${project.parent.groupId}:clickhouse-grpc-client - - ** - - - - ${project.parent.groupId}:io.grpc - - ** - - - - ${project.parent.groupId}:org.roaringbitmap - - ** - - - - com.google.code.gson:gson - - ** - - *:* - mozilla/** **/module-info.class - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - META-INF/*.xml @@ -292,6 +397,7 @@ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> ${project.groupId}.jdbc + ${mainClass} ${spec.title} ${spec.version} @@ -302,15 +408,9 @@ *:* google/** - mozilla/** org/checkerframework/** org/codehaus/** **/module-info.class - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native-image/** - META-INF/*.xml @@ -328,6 +428,17 @@ true true grpc + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + com.clickhouse:clickhouse-grpc-client + com.clickhouse:io.grpc + io.grpc:grpc-netty-shaded + io.grpc:grpc-okhttp + org.apache.commons:commons-compress + + com.google @@ -345,10 +456,6 @@ io.perfmark ${shade.base}.perfmark - - net.jpountz - ${shade.base}.jpountz - okio ${shade.base}.okio @@ -369,41 +476,20 @@ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> ${project.groupId}.jdbc + ${mainClass} ${spec.title} ${spec.version} - - ${project.parent.groupId}:clickhouse-cli-client - - ** - - - - ${project.parent.groupId}:clickhouse-http-client - - ** - - *:* google/** - mozilla/** org/checkerframework/** org/codehaus/** - **/darwin/** - **/linux/** - **/win32/** **/module-info.class - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - META-INF/*.xml @@ -421,6 +507,14 @@ true true http + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + com.clickhouse:clickhouse-http-client + org.lz4:lz4-pure-java + + net.jpountz @@ -428,70 +522,21 @@ - - - ${project.groupId}.jdbc + ${mainClass} ${spec.title} ${spec.version} - - ${project.parent.groupId}:clickhouse-cli-client - - ** - - - - ${project.parent.groupId}:clickhouse-grpc-client - - ** - - - - ${project.parent.groupId}:io.grpc - - ** - - - - ${project.parent.groupId}:org.roaringbitmap - - ** - - - - com.google.code.gson:gson - - ** - - *:* - google/** - mozilla/** - org/apache/** - org/checkerframework/** - org/codehaus/** - **/darwin/** - **/linux/** - **/win32/** **/module-info.class - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - META-INF/*.xml @@ -509,6 +554,14 @@ true true cli + + + com.clickhouse:clickhouse-data + com.clickhouse:clickhouse-client + com.clickhouse:clickhouse-cli-client + org.lz4:lz4-pure-java + + net.jpountz @@ -516,70 +569,21 @@ - - - ${project.groupId}.jdbc + ${mainClass} ${spec.title} ${spec.version} - - ${project.parent.groupId}:clickhouse-grpc-client - - ** - - - - ${project.parent.groupId}:clickhouse-http-client - - ** - - - - ${project.parent.groupId}:io.grpc - - ** - - - - ${project.parent.groupId}:org.roaringbitmap - - ** - - - - com.google.code.gson:gson - - ** - - *:* - google/** - mozilla/** - org/apache/** - org/checkerframework/** - org/codehaus/** - **/darwin/** - **/linux/** - **/win32/** **/module-info.class - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - META-INF/*.xml diff --git a/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Main.java b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Main.java new file mode 100644 index 000000000..c559d5e77 --- /dev/null +++ b/clickhouse-jdbc/src/main/java/com/clickhouse/jdbc/Main.java @@ -0,0 +1,813 @@ +package com.clickhouse.jdbc; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Objects; + +import com.clickhouse.client.ClickHouseClient; +import com.clickhouse.client.ClickHouseConfig; +import com.clickhouse.client.ClickHouseException; +import com.clickhouse.client.ClickHouseNode; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.ClickHouseRequest.Mutation; +import com.clickhouse.client.config.ClickHouseClientOption; +import com.clickhouse.data.ClickHouseColumn; +import com.clickhouse.data.ClickHouseDataProcessor; +import com.clickhouse.data.ClickHouseDataStreamFactory; +import com.clickhouse.data.ClickHouseDataType; +import com.clickhouse.data.ClickHouseDeserializer; +import com.clickhouse.data.ClickHouseFormat; +import com.clickhouse.data.ClickHouseInputStream; +import com.clickhouse.data.ClickHouseOutputStream; +import com.clickhouse.data.ClickHouseRecord; +import com.clickhouse.data.ClickHouseSerializer; +import com.clickhouse.data.ClickHouseValue; +import com.clickhouse.data.format.BinaryStreamUtils; +import com.clickhouse.data.value.ClickHouseByteValue; +import com.clickhouse.data.value.ClickHouseLongValue; +import com.clickhouse.data.value.ClickHouseStringValue; +import com.clickhouse.jdbc.internal.ClickHouseConnectionImpl; + +public final class Main { + static class Options { + final String action; + final int batch; + final boolean output; + final int samples; + final boolean serde; + final String type; + final boolean verbose; + + final String url; + final String query; + final String file; + + final boolean requiresJdbc; + + private Options(String url, String query, String file) { + action = System.getProperty("action", "read").toLowerCase(); + batch = Integer.getInteger("batch", 1000); + output = Boolean.getBoolean("output"); + samples = Integer.getInteger("samples", 500000000); + serde = !"false".equalsIgnoreCase(System.getProperty("serde", "")); + type = System.getProperty("type", "").toLowerCase(); + verbose = Boolean.getBoolean("verbose"); + + this.url = url; + this.requiresJdbc = url.length() > 5 && "jdbc:".equalsIgnoreCase(url.substring(0, 5)); + + if (query == null || query.isEmpty()) { + this.query = isLoadAction() || isWriteAction() ? getInsertQuery() : getSelectQuery(); + } else { + this.query = query; + } + if (file == null || file.isEmpty()) { + this.file = requiresJdbc ? "jdbc.out" : "java.out"; + } else { + this.file = file; + } + + if (verbose) { + println("Arguments:"); + println(" - url=%s", this.url); + println(" - query=%s", this.query); + println(" - file=%s", this.file); + println(); + println("Options: action=%s, batch=%d, samples=%d, serde=%s, type=%s, verbose=%s", action, batch, + samples, serde, type, verbose); + } + } + + int getSamples() { + final int s; + if (isMixed() || isTuple() || isNested()) { + s = samples / 5; + } else if (isArray()) { + s = samples / 1000; + } else if (isJson()) { + s = samples / 500; + } else { + s = samples; + } + return s; + } + + boolean isDumpAction() { + return "dump".equals(action); + } + + boolean isLoadAction() { + return "load".equals(action); + } + + boolean isWriteAction() { + return "write".equals(action); + } + + boolean isInt8() { + return "int8".equals(type); + } + + boolean isUInt64() { + return "uint64".equals(type); + } + + boolean isString() { + return "string".equals(type); + } + + boolean isDateTime() { + return "datetime".equals(type); + } + + boolean isDecimal() { + return "decimal".equals(type); + } + + boolean isMixed() { + return "mixed".equals(type); + } + + boolean isArray() { + return "array".equals(type); + } + + boolean isTuple() { + return "tuple".equals(type); + } + + boolean isNested() { + return "nested".equals(type); + } + + boolean isJson() { + return "json".equals(type); + } + + List getColumns() { + final List columns; + if (isInt8()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Int8, false)); + } else if (isUInt64()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.UInt64, false)); + } else if (isString()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.String, false)); + } else if (isDateTime()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.DateTime, false)); + } else if (isDecimal()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Decimal128, false, 0, 6)); + } else if (isMixed()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Int8, false), + ClickHouseColumn.of(null, ClickHouseDataType.UInt64, false), + ClickHouseColumn.of(null, ClickHouseDataType.String, false), + ClickHouseColumn.of(null, ClickHouseDataType.DateTime, false), + ClickHouseColumn.of(null, ClickHouseDataType.Decimal128, false, 0, 6)); + } else if (isArray()) { + columns = Arrays.asList(ClickHouseColumn.of(null, "Array(Int32)")); + } else if (isTuple()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Tuple, false, + ClickHouseColumn.of(null, ClickHouseDataType.Int8, false), + ClickHouseColumn.of(null, ClickHouseDataType.UInt64, false), + ClickHouseColumn.of(null, ClickHouseDataType.String, false), + ClickHouseColumn.of(null, ClickHouseDataType.DateTime, false), + ClickHouseColumn.of(null, ClickHouseDataType.Decimal128, false, 0, 6))); + } else if (isNested()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Nested, false, + ClickHouseColumn.of(null, ClickHouseDataType.Int8, false), + ClickHouseColumn.of(null, ClickHouseDataType.UInt64, false), + ClickHouseColumn.of(null, ClickHouseDataType.String, false), + ClickHouseColumn.of(null, ClickHouseDataType.DateTime, false), + ClickHouseColumn.of(null, ClickHouseDataType.Decimal128, false, 0, 6))); + } else if (isJson()) { + columns = Arrays.asList(ClickHouseColumn.of(null, ClickHouseDataType.Tuple, false, + ClickHouseColumn.of(null, ClickHouseDataType.Int8, false), + ClickHouseColumn.of(null, ClickHouseDataType.UInt64, false), + ClickHouseColumn.of(null, ClickHouseDataType.String, false), + ClickHouseColumn.of(null, ClickHouseDataType.DateTime, false), + ClickHouseColumn.of(null, ClickHouseDataType.Decimal128, false, 0, 6))); + } else { + columns = null; + } + return columns; + } + + ClickHouseDeserializer getDerializer(ClickHouseConfig config) throws IOException { + final List columns = getColumns(); + if (columns == null || columns.isEmpty()) { + throw new IllegalStateException("Not column information available for query: " + query); + } + + final ClickHouseDataProcessor processor = ClickHouseDataStreamFactory.getInstance().getProcessor(config, + null, ClickHouseOutputStream.empty(), null, columns); + final ClickHouseDeserializer[] deserializers = processor.getDeserializers(config, columns); + return deserializers.length == 1 ? deserializers[0] + : ClickHouseDeserializer.of(Arrays.asList(deserializers)); + } + + ClickHouseSerializer getSerializer(ClickHouseConfig config) throws IOException { + final List columns = getColumns(); + if (columns == null || columns.isEmpty()) { + throw new IllegalStateException("Not column information available for query: " + query); + } + + final ClickHouseDataProcessor processor = ClickHouseDataStreamFactory.getInstance().getProcessor(config, + null, ClickHouseOutputStream.empty(), null, columns); + final ClickHouseSerializer[] serializers = processor.getSerializers(config, columns); + return serializers.length == 1 ? serializers[0] : ClickHouseSerializer.of(Arrays.asList(serializers)); + } + + String getSelectQuery() { + final String selectQuery; + if (isInt8()) { + selectQuery = "select number::Int8 v from numbers(%d)"; + } else if (isUInt64()) { + selectQuery = "select number v from numbers(%d)"; + } else if (isString()) { + selectQuery = "select toString(number) v from numbers(%d)"; + } else if (isDateTime()) { + selectQuery = "select toDateTime(number) v from numbers(%d)"; + } else if (isDecimal()) { + selectQuery = "select toDecimal128(number, 6) v from numbers(%d)"; + } else if (isMixed()) { + selectQuery = "select number::Int8 a, number b, toString(number) c, toDateTime(number) d, toDecimal128(number, 6) e from numbers(%d)"; + } else if (isArray()) { + selectQuery = "select range(100000, 101000 + number % 1000) v from numbers(%d)"; + } else if (isTuple()) { + selectQuery = "select tuple(number::Int8, number, toString(number), toDateTime(number), toDecimal128(number, 6)) v from numbers(%d)"; + } else if (isNested()) { + selectQuery = "select [(number::Int8, number, toString(number), toDateTime(number), toDecimal128(number, 6))]::Nested(a Int8, b UInt64, c String, d DateTime, e Decimal128(6)) v from numbers(%d)"; + } else if (isJson()) { + selectQuery = "select (number::Int8, number, toString(number), toDateTime(number), toDecimal128(number, 6), range(1000,1005), [tuple(number, number+1)])::Tuple(a Int8, b UInt64, c String, d DateTime, e Decimal128(6), f Array(UInt16), g Nested(x UInt64, y UInt64)) v from numbers(%d)"; + } else { + selectQuery = "select %d"; + } + return String.format(selectQuery, getSamples()); + } + + String getInsertQuery() { + return type.isEmpty() ? "insert into test_insert" : "insert into test_insert_" + type; + } + } + + static class GenericQuery { + static final ClickHouseFormat defaultFormat = ClickHouseFormat.RowBinaryWithNamesAndTypes; + + protected final Options options; + + protected GenericQuery(Options options) { + this.options = options; + } + + final long run() throws ClickHouseException, SQLException { + final long rows; + if (options.isDumpAction()) { + rows = dump(); + } else if (options.isLoadAction()) { + rows = load(options); + } else if (options.isWriteAction()) { + rows = write(options); + } else { + rows = read(options); + } + return rows; + } + + long read(ResultSet rs) throws SQLException { + long count = 0L; + final int len = rs.getMetaData().getColumnCount(); + while (rs.next()) { + Object obj = null; + for (int i = 1; i <= len; i++) { + // autoboxing to ensure we "got" the value + obj = rs.getObject(i); + } + if (obj != null) { + count++; + } + } + return count; + } + + long read(ClickHouseResponse response) throws ClickHouseException { + long count = 0L; + int len = response.getColumns().size(); + for (ClickHouseRecord r : response.records()) { + Object obj = null; + for (int i = 0; i < len; i++) { + // autoboxing just for comparison + obj = r.getValue(i).asObject(); + } + if (obj != null) { + count++; + } + } + return count; + } + + long write(Connection conn) throws SQLException { + throw new UnsupportedOperationException("No idea how to write data for custom query"); + } + + long write(Mutation request) throws ClickHouseException { + throw new UnsupportedOperationException("No idea how to write data for custom query"); + } + + final long dump() throws ClickHouseException, SQLException { + final long rows; + if (options.requiresJdbc) { + try (ClickHouseConnection conn = new ClickHouseConnectionImpl(options.url)) { + ClickHouseRequest request = conn.unwrap(ClickHouseRequest.class).query(options.query); + if (!request.getServer().getConfig().hasOption(ClickHouseClientOption.FORMAT)) { + request.format(defaultFormat.defaultInputFormat()); + } + request.output(options.file); + try (ClickHouseResponse response = request.executeAndWait()) { + rows = response.getSummary().getReadRows(); + } + } + } else { // java client + final ClickHouseNode server = ClickHouseNode.of(options.url); + try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol())) { + ClickHouseRequest request = client.read(server).query(options.query); + if (!server.getConfig().hasOption(ClickHouseClientOption.FORMAT)) { + request.format(defaultFormat.defaultInputFormat()); + } + request.output(options.file); + try (ClickHouseResponse response = request.query(options.query).executeAndWait()) { + rows = response.getSummary().getReadRows(); + } + } + } + return rows; + } + + final long load(Options options) throws ClickHouseException, SQLException { + final long rows; + if (options.requiresJdbc) { + try (ClickHouseConnection conn = new ClickHouseConnectionImpl(options.url)) { + ClickHouseFormat format = conn.getConfig().getFormat(); + if (!conn.unwrap(ClickHouseRequest.class).getServer().getConfig() + .hasOption(ClickHouseClientOption.FORMAT)) { + format = defaultFormat.defaultInputFormat(); + } + try (PreparedStatement stmt = conn.prepareStatement(options.query + " format " + format.name())) { + stmt.setObject(1, new File(options.file)); + rows = stmt.executeLargeUpdate(); + } + } + } else { // java client + final ClickHouseNode server = ClickHouseNode.of(options.url); + try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol())) { + Mutation request = client.write(server).data(options.file); + if (!server.getConfig().hasOption(ClickHouseClientOption.FORMAT)) { + request.format(defaultFormat.defaultInputFormat()); + } + try (ClickHouseResponse response = request + .query(options.query + " format " + request.getConfig().getFormat().name()) + .executeAndWait()) { + rows = response.getSummary().getWrittenRows(); + } + } + } + return rows; + } + + final long read(Options options) throws ClickHouseException, SQLException { + final long rows; + if (options.requiresJdbc) { + try (ClickHouseConnection conn = new ClickHouseConnectionImpl(options.url); + Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery(options.query)) { + try { + rs.unwrap(ClickHouseResponse.class).getInputStream() + .setCopyToTarget(new FileOutputStream(options.file, false)); + } catch (IOException e) { + throw SqlExceptionUtils.clientError(e); + } + rows = read(rs); + } + } + } else { // java client + final ClickHouseNode server = ClickHouseNode.of(options.url); + try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol())) { + ClickHouseRequest request = client.read(server).query(options.query); + if (!server.getConfig().hasOption(ClickHouseClientOption.FORMAT)) { + request.format(defaultFormat); + } + try (ClickHouseResponse response = request.executeAndWait()) { + try { + response.getInputStream().setCopyToTarget(new FileOutputStream(options.file, false)); + } catch (IOException e) { + throw ClickHouseException.of(e, server); + } + rows = read(response); + } + } + } + return rows; + } + + final long write(Options options) throws ClickHouseException, SQLException { + final long rows; + if (options.requiresJdbc) { + try (ClickHouseConnection conn = new ClickHouseConnectionImpl(options.url)) { + rows = write(conn); + } + } else { // java client + final ClickHouseNode server = ClickHouseNode.of(options.url); + try (ClickHouseClient client = ClickHouseClient.newInstance(server.getProtocol())) { + Mutation request = client.write(server).query(options.query).data(options.file); + if (!server.getConfig().hasOption(ClickHouseClientOption.FORMAT)) { + request.format(defaultFormat.defaultInputFormat()); + } + rows = write(request); + } + } + return rows; + } + } + + static class Int8Query extends GenericQuery { + Int8Query(Options options) { + super(options); + } + + @Override + long read(ResultSet rs) throws SQLException { + long count = 0L; + final int len = rs.getMetaData().getColumnCount(); + byte v = (byte) 0; + while (rs.next()) { + for (int i = 1; i <= len; i++) { + v = rs.getByte(i); + } + count++; + } + long lastValue = 0xFFL & v; + return count >= lastValue ? count : lastValue; + } + + @Override + long read(ClickHouseResponse response) throws ClickHouseException { + long count = 0L; + byte v = (byte) 0; + if (options.serde) { + if (options.verbose) { + println("Deserialization: records"); + } + for (ClickHouseRecord r : response.records()) { + // only one column + v = r.getValue(0).asByte(); + count++; + } + } else { + if (options.verbose) { + println("Deserialization: readByte"); + } + try (ClickHouseInputStream in = response.getInputStream()) { + for (long i = 0L, len = options.samples; i < len; i++) { + v = in.readByte(); + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + long lastValue = 0xFFL & v; + return count >= lastValue ? count : lastValue; + } + + @Override + long write(Connection conn) throws SQLException { + try (PreparedStatement stmt = conn.prepareStatement(options.query)) { + final int batchSize = options.batch; + long count = 0L; + long rows = 0L; + for (long i = 0, len = options.getSamples(); i < len; i++) { + stmt.setByte(1, (byte) i); + stmt.addBatch(); + if ((count = (i + 1) % batchSize) == 0L) { + rows += stmt.executeLargeBatch().length; + } + } + if (count > 0L) { + rows += stmt.executeLargeBatch().length; + } + return rows; + } + } + + @Override + long write(Mutation request) throws ClickHouseException { + try (ClickHouseResponse response = request.data(o -> { + if (options.serde) { + ClickHouseConfig config = request.getConfig(); + ClickHouseSerializer serializer = options.getSerializer(config); + ClickHouseValue value = ClickHouseByteValue.ofNull(); + if (options.verbose) { + println("Serialization: %s -> %s", serializer, value); + } + for (long i = 0L, len = options.samples; i < len; i++) { + serializer.serialize(value.update(i), o); + } + } else { + if (options.verbose) { + println("Serialization: writeByte"); + } + for (long i = 0L, len = options.samples; i < len; i++) { + o.writeByte((byte) i); + } + } + }).executeAndWait()) { + return response.getSummary().getWrittenRows(); + } + } + } + + static class UInt64Query extends GenericQuery { + UInt64Query(Options options) { + super(options); + } + + @Override + long read(ResultSet rs) throws SQLException { + long count = 0L; + final int len = rs.getMetaData().getColumnCount(); + long v = 0L; + while (rs.next()) { + for (int i = 1; i <= len; i++) { + v = rs.getLong(i); + } + count++; + } + return count >= v ? count : v; + } + + @Override + long read(ClickHouseResponse response) throws ClickHouseException { + long count = 0L; + long v = 0L; + if (options.serde) { + if (options.verbose) { + println("Deserialization: records"); + } + for (ClickHouseRecord r : response.records()) { + // only one column + v = r.getValue(0).asLong(); + count++; + } + } else { + if (options.verbose) { + println("Deserialization: readByte"); + } + try (ClickHouseInputStream in = response.getInputStream()) { + for (long i = 0L, len = options.samples; i < len; i++) { + v = in.readBuffer(8).asLong(); + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return count >= v ? count : v; + } + + @Override + long write(Connection conn) throws SQLException { + try (PreparedStatement stmt = conn.prepareStatement(options.query)) { + final int batchSize = options.batch; + long count = 0L; + long rows = 0L; + for (long i = 0, len = options.getSamples(); i < len; i++) { + stmt.setLong(1, i); + stmt.addBatch(); + if ((count = (i + 1) % batchSize) == 0L) { + rows += stmt.executeLargeBatch().length; + } + } + if (count > 0L) { + rows += stmt.executeLargeBatch().length; + } + return rows; + } + } + + @Override + long write(Mutation request) throws ClickHouseException { + try (ClickHouseResponse response = request.data(o -> { + if (options.serde) { + ClickHouseConfig config = request.getConfig(); + ClickHouseSerializer serializer = options.getSerializer(config); + ClickHouseValue value = ClickHouseLongValue.ofUnsignedNull(); + if (options.verbose) { + println("Serialization: %s -> %s", serializer, value); + } + for (long i = 0L, len = options.samples; i < len; i++) { + serializer.serialize(value.update(i), o); + } + } else { + if (options.verbose) { + println("Serialization: writeLong"); + } + for (long i = 0L, len = options.samples; i < len; i++) { + BinaryStreamUtils.writeUnsignedInt64(o, i); + } + } + }).executeAndWait()) { + return response.getSummary().getWrittenRows(); + } + } + } + + static class StringQuery extends GenericQuery { + StringQuery(Options options) { + super(options); + } + + @Override + long read(ResultSet rs) throws SQLException { + long count = 0L; + final int len = rs.getMetaData().getColumnCount(); + String v = null; + while (rs.next()) { + for (int i = 1; i <= len; i++) { + v = rs.getString(i); + } + count++; + } + return v != null ? count : 0L; + } + + @Override + long read(ClickHouseResponse response) throws ClickHouseException { + long count = 0L; + String v = null; + if (options.serde) { + if (options.verbose) { + println("Deserialization: records"); + } + for (ClickHouseRecord r : response.records()) { + // only one column + v = r.getValue(0).asString(); + count++; + } + } else { + if (options.verbose) { + println("Deserialization: readByte"); + } + try (ClickHouseInputStream in = response.getInputStream()) { + for (long i = 0L, len = options.samples; i < len; i++) { + v = in.readUnicodeString(); + count++; + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + return v != null ? count : 0L; + } + + @Override + long write(Connection conn) throws SQLException { + try (PreparedStatement stmt = conn.prepareStatement(options.query)) { + final int batchSize = options.batch; + long count = 0L; + long rows = 0L; + for (long i = 0, len = options.getSamples(); i < len; i++) { + stmt.setString(1, Long.toString(i)); + stmt.addBatch(); + if ((count = (i + 1) % batchSize) == 0L) { + rows += stmt.executeLargeBatch().length; + } + } + if (count > 0L) { + rows += stmt.executeLargeBatch().length; + } + return rows; + } + } + + @Override + long write(Mutation request) throws ClickHouseException { + try (ClickHouseResponse response = request.data(o -> { + if (options.serde) { + ClickHouseConfig config = request.getConfig(); + ClickHouseSerializer serializer = options.getSerializer(config); + ClickHouseValue value = ClickHouseStringValue.ofNull(); + if (options.verbose) { + println("Serialization: %s -> %s", serializer, value); + } + for (long i = 0L, len = options.samples; i < len; i++) { + serializer.serialize(value.update(i), o); + } + } else { + if (options.verbose) { + println("Serialization: writeString"); + } + for (long i = 0L, len = options.samples; i < len; i++) { + o.writeUnicodeString(Long.toString(i)); + } + } + }).executeAndWait()) { + return response.getSummary().getWrittenRows(); + } + } + } + + private static void println() { + System.out.println(); // NOSONAR + } + + private static void println(Object msg, Object... args) { + if (args == null || args.length == 0) { + System.out.println(msg); // NOSONAR + } else { + System.out.println(String.format(Locale.ROOT, Objects.toString(msg), args)); // NOSONAR + } + } + + private static void printUsage() { + String execFile = "clickhouse-jdbc-bin"; + try { + File file = Paths.get(Main.class.getProtectionDomain().getCodeSource().getLocation().toURI()) + .toFile(); + if (file.isFile()) { + execFile = file.getName(); + if (!Files.isExecutable(file.toPath())) { + execFile = "java -jar " + execFile; + } + } else { + execFile = "java -cp " + file.getCanonicalPath() + " " + Main.class.getName(); + } + } catch (Exception e) { + // ignore + } + + final int index = execFile.indexOf(' '); + println("Usage: %s [QUERY] [FILE]", + index > 0 ? (execFile.substring(0, index) + " [PROPERTIES]" + execFile.substring(index)) + : (execFile + " [PROPERTIES]")); + println(); + println("Properties: -Dkey=value [-Dkey=value]*"); + println(" action \tAction, one of read(default), write, dump(no deserialization), and load(no serialization)"); + println(" batch \tBatch size for JDBC writing, defaults to 1000"); + println(" output \tWhether to write raw response into a file(java.out or jdbc.out), defaults to false"); + println(" samples\tSamples, defaults to 500000000"); + println(" serde \tWhether to use default serialization/deserializion mechanism in Java client, defaults to true"); + println(" type \tPredefined QUERY, one of Int8, UInt64, String, Array, Tuple, Nested, and Mixed"); + println(" verbose\tWhether to show logs, defaults to false"); + println(); + println("Examples:"); + println(" - %s 'https://localhost?sslmode=none' 'select 1'", + index > 0 ? (execFile.substring(0, index) + " -Dverbose=true" + execFile.substring(index)) + : (execFile + " -Dverbose=true")); + println(" - %s 'jdbc:ch://user:password@localhost:8123/default' 'select 1' output.file", execFile); + println(" - %s 'jdbc:ch:http://node1,node2,node3/default' 'insert into table1' input.file", execFile); + } + + public static void main(String[] args) throws Exception { + if ((args == null || args.length < 1) || args.length > 3) { + printUsage(); + System.exit(1); + } + + final Options options = new Options(args[0].trim(), args.length > 1 ? args[1].trim() : null, + args.length > 2 ? args[2].trim() : null); + + final GenericQuery query; + if (options.isInt8()) { + query = new Int8Query(options); + } else if (options.isUInt64()) { + query = new UInt64Query(options); + } else if (options.isString()) { + query = new StringQuery(options); + } else { + query = new GenericQuery(options); + } + + final long startTime = options.verbose ? System.nanoTime() : 0L; + final long rows = query.run(); + if (options.verbose) { + long elapsedNanos = System.nanoTime() - startTime; + println("Processed %,d rows in %,.2f ms (%,.2f rows/s)", rows, elapsedNanos / 1_000_000D, + rows * 1_000_000_000D / elapsedNanos); + } + System.exit(rows > 0L ? 0 : 1); + } + + private Main() { + } +} \ No newline at end of file diff --git a/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/native-image.properties b/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/native-image.properties new file mode 100755 index 000000000..98d306764 --- /dev/null +++ b/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/native-image.properties @@ -0,0 +1 @@ +Args = -H:ReflectionConfigurationResources=${.}/reflect-config.json diff --git a/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/reflect-config.json b/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/reflect-config.json new file mode 100644 index 000000000..bcc6aad78 --- /dev/null +++ b/clickhouse-jdbc/src/main/resources/META-INF/native-image/com.clickhouse/clickhouse-jdbc/reflect-config.json @@ -0,0 +1,36 @@ +[ + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4HCJNICompressor", + "fields": [{ "name": "INSTANCE" }], + "methods": [{ "name": "", "parameterTypes": ["int"] }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4HCJavaUnsafeCompressor", + "fields": [{ "name": "INSTANCE" }], + "methods": [{ "name": "", "parameterTypes": ["int"] }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JNICompressor", + "fields": [{ "name": "INSTANCE" }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JNIFastDecompressor", + "fields": [{ "name": "INSTANCE" }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JNISafeDecompressor", + "fields": [{ "name": "INSTANCE" }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JavaUnsafeCompressor", + "fields": [{ "name": "INSTANCE" }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JavaUnsafeFastDecompressor", + "fields": [{ "name": "INSTANCE" }] + }, + { + "name": "com.clickhouse.client.internal.jpountz.lz4.LZ4JavaUnsafeSafeDecompressor", + "fields": [{ "name": "INSTANCE" }] + } +] diff --git a/pom.xml b/pom.xml index cb834bb50..3aebcc4fc 100644 --- a/pom.xml +++ b/pom.xml @@ -100,6 +100,7 @@ 1.22 3.5.2 8.5.11 + 20.0 1.52.1 2.10.1 4.0.1 @@ -137,6 +138,7 @@ 0.8.8 3.2.2 3.4.0 + 0.9.20 1.7.0 0.6.1 3.3.0 @@ -217,6 +219,11 @@ zstd-jni ${zstd-jni.version} + + com.graphql-java + graphql-java + ${graphql.version} + dnsjava dnsjava @@ -290,6 +297,11 @@ lz4-java ${lz4.version} + + org.lz4 + lz4-pure-java + ${lz4.version} + org.msgpack msgpack-core @@ -335,6 +347,11 @@ httpclient5 ${apache.httpclient.version} + + org.apache.httpcomponents.core5 + httpcore5 + ${apache.httpclient.version} + org.mariadb.jdbc @@ -516,6 +533,11 @@ versions-maven-plugin ${versions-plugin.version} + + org.graalvm.buildtools + native-maven-plugin + ${native-plugin.version} + org.sonatype.plugins nexus-staging-maven-plugin @@ -919,6 +941,7 @@ default-jar + false true true @@ -929,7 +952,7 @@ ${project.url} ClickHouse, Inc. ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + ${git.commit.id.full} @@ -1141,6 +1164,7 @@ default-jar + false true true @@ -1151,7 +1175,7 @@ ${project.url} ClickHouse, Inc. ${project.groupId} - ${project.artifactId} ${project.version} (revision: ${git.commit.id.abbrev}) + ${git.commit.id.full}