From 253e1a3b9a9249e8ac44f4db9d55a5e3bcb932ef Mon Sep 17 00:00:00 2001 From: Zhichun Wu Date: Mon, 29 Aug 2022 19:40:59 +0800 Subject: [PATCH] Update JDBC examples --- examples/jdbc/pom.xml | 4 +- .../com/clickhouse/examples/jdbc/Advance.java | 79 ------- .../clickhouse/examples/jdbc/Advanced.java | 192 ++++++++++++++++++ .../com/clickhouse/examples/jdbc/Basic.java | 172 +++++++++++----- 4 files changed, 312 insertions(+), 135 deletions(-) delete mode 100644 examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advance.java create mode 100644 examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java diff --git a/examples/jdbc/pom.xml b/examples/jdbc/pom.xml index b7fe4ad9e..5db22317f 100644 --- a/examples/jdbc/pom.xml +++ b/examples/jdbc/pom.xml @@ -7,7 +7,7 @@ 1.0.0 jar - jdbc-examples + JDBC Examples JDBC Examples https://github.com/ClickHouse/clickhouse-jdbc 2022 @@ -56,7 +56,7 @@ UTF-8 UTF-8 - 0.3.2-patch9 + 0.3.3-SNAPSHOT 3.8.1 diff --git a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advance.java b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advance.java deleted file mode 100644 index b092aae03..000000000 --- a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advance.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.clickhouse.examples.jdbc; - -import java.io.ByteArrayInputStream; -import java.nio.charset.StandardCharsets; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.util.Properties; - -import com.clickhouse.client.ClickHouseFormat; -import com.clickhouse.client.data.ClickHouseExternalTable; - -public class Advance { - static String exteralTables(String url, String user, String password) throws SQLException { - String sql = "select a.name as n1, b.name as n2 from {tt 'table1'} a inner join {tt 'table2'} b on a.id=b.id"; - try (Connection conn = DriverManager.getConnection(url, user, password); - PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setObject(1, - ClickHouseExternalTable.builder().name("table1").columns("id Int32, name Nullable(String)") - .format(ClickHouseFormat.CSV) - .content(new ByteArrayInputStream("1,a\n2,b".getBytes(StandardCharsets.US_ASCII))).build()); - ps.setObject(2, - ClickHouseExternalTable.builder().name("table2").columns("id Int32, name String") - .format(ClickHouseFormat.JSONEachRow) - .content(new ByteArrayInputStream("{\"id\":3,\"name\":\"c\"}\n{\"id\":1,\"name\":\"d\"}" - .getBytes(StandardCharsets.US_ASCII))) - .build()); - try (ResultSet rs = ps.executeQuery()) { - if (!rs.next()) { - throw new IllegalStateException("Should have at least one record"); - } - - // n1=a, n2=d - return String.format("n1=%s, n2=%s", rs.getString(1), rs.getString(2)); - } - } - } - - static String namedParameter(String url, String user, String password) throws SQLException { - Properties props = new Properties(); - props.setProperty("user", user); - props.setProperty("password", password); - props.setProperty("namedParameter", "true"); - // two parameters: - // * a - String - // * b - DateTime64(3) - String sql = "select :a as a1, :a(String) as a2, :b(DateTime64(3)) as b"; - try (Connection conn = DriverManager.getConnection(url, props); - PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setString(1, "a"); - ps.setObject(2, LocalDateTime.of(2022, 1, 7, 22, 48, 17, 123000000)); - - try (ResultSet rs = ps.executeQuery()) { - if (!rs.next()) { - throw new IllegalStateException("Should have at least one record"); - } - // a1=a, a2=a, b=2022-01-07 22:48:17.123 - return String.format("a1=%s, a2=%s, b=%s", rs.getString(1), rs.getString(2), rs.getString("B")); - } - } - } - - public static void main(String[] args) { - String url = String.format("jdbc:ch://%s:%d/system", System.getProperty("chHost", "localhost"), - Integer.parseInt(System.getProperty("chPort", "8123"))); - String user = System.getProperty("chUser", "default"); - String password = System.getProperty("chPassword", ""); - - try { - exteralTables(url, user, password); - namedParameter(url, user, password); - } catch (SQLException e) { - e.printStackTrace(); - } - } -} diff --git a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java new file mode 100644 index 000000000..ef9d74fd1 --- /dev/null +++ b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Advanced.java @@ -0,0 +1,192 @@ +package com.clickhouse.examples.jdbc; + +import java.io.ByteArrayInputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; + +import com.clickhouse.client.ClickHouseException; +import com.clickhouse.client.ClickHouseFormat; +import com.clickhouse.client.ClickHouseRecord; +import com.clickhouse.client.ClickHouseRequest; +import com.clickhouse.client.ClickHouseRequestManager; +import com.clickhouse.client.ClickHouseResponse; +import com.clickhouse.client.data.ClickHouseExternalTable; +import com.clickhouse.jdbc.ClickHouseConnection; +import com.clickhouse.jdbc.SqlExceptionUtils; + +public class Advanced { + private static Connection getConnection(String url, Properties props) throws SQLException { + Connection conn = DriverManager.getConnection(url, props); + System.out.println("Connected to: " + conn.getMetaData().getURL()); + return conn; + } + + private static Connection getConnection(String url) throws SQLException { + return getConnection(url, new Properties()); + } + + private static ByteArrayInputStream newInputStream(String content) { + return new ByteArrayInputStream(content.getBytes(StandardCharsets.US_ASCII)); + } + + static String customQueryId(String url) throws SQLException { + String sql = "select 1"; + String queryId = "my-query-id"; + String result = ""; + try (Connection conn = getConnection(url); Statement stmt = conn.createStatement()) { + stmt.unwrap(ClickHouseRequest.class).manager(new ClickHouseRequestManager() { + private final AtomicInteger id = new AtomicInteger(0); + + @Override + public String createQueryId() { + return "my-query-" + id.incrementAndGet(); + } + }); + try (ClickHouseResponse resp = stmt.unwrap(ClickHouseRequest.class).query(sql, queryId).executeAndWait()) { + result = resp.firstRecord().getValue(1).asString(); + } catch (ClickHouseException | UncheckedIOException e) { + throw SqlExceptionUtils.handle(e); + } + } + return result; + } + + static String exteralTables(String url) throws SQLException { + String sql = "select a.name as n1, b.name as n2 from {tt 'table1'} a inner join {tt 'table2'} b on a.id=b.id"; + try (Connection conn = getConnection(url); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setObject(1, + ClickHouseExternalTable.builder().name("table1").columns("id Int32, name Nullable(String)") + .format(ClickHouseFormat.CSV) + .content(newInputStream("1,a\n2,b")).build()); + ps.setObject(2, + ClickHouseExternalTable.builder().name("table2").columns("id Int32, name String") + .format(ClickHouseFormat.JSONEachRow) + .content(newInputStream("{\"id\":3,\"name\":\"c\"}\n{\"id\":1,\"name\":\"d\"}")).build()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + throw new IllegalStateException("Should have at least one record"); + } + + // n1=a, n2=d + return String.format("n1=%s, n2=%s", rs.getString(1), rs.getString(2)); + } + } + } + + static String manualTransaction(String url) throws SQLException { + Properties props = new Properties(); + // props.setProperty(JdbcConfig.PROP_AUTO_COMMIT, "false"); + props.setProperty("autoCommit", "false"); + // props.setProperty(JdbcConfig.PROP_TX_SUPPORT, "true"); + props.setProperty("transactionSupport", "true"); + try (Connection conn = getConnection(url, props)) { + if (!((ClickHouseConnection) conn).isTransactionSupported()) { + System.out.println("Re-establishing connection until transaction is supported..."); + return manualTransaction(url); + } + + conn.commit(); + return "Transaction committed!"; + } + } + + static String namedParameter(String url) throws SQLException { + Properties props = new Properties(); + // props.setProperty(JdbcConfig.PROP_NAMED_PARAM, "true"); + props.setProperty("namedParameter", "true"); + // two parameters: + // * a - String + // * b - DateTime64(3) + String sql = "select :a as a1, :a(String) as a2, :b(DateTime64(3)) as b"; + try (Connection conn = getConnection(url, props); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, "a"); + ps.setObject(2, LocalDateTime.of(2022, 1, 7, 22, 48, 17, 123000000)); + + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + throw new IllegalStateException("Should have at least one record"); + } + + // a1=a, a2=a, b=2022-01-07 22:48:17.123 + return String.format("a1=%s, a2=%s, b=%s", rs.getString(1), rs.getString(2), rs.getString("B")); + } + } + } + + static String renameResponseColumn(String url) throws SQLException { + Properties props = new Properties(); + // props.setProperty(ClickHouseClientOption.RENAME_RESPONSE_COLUMN.getKey(), + // ClickHouseRenameMethod.TO_CAMELCASE_WITHOUT_PREFIX.name()); + props.setProperty("rename_response_column", "TO_CAMELCASE_WITHOUT_PREFIX"); + + String sql = "SELECT c.`simple.string_value`, n.number\n" + + "FROM (SELECT 1 number, 'string' `simple.string_value`) c\n" + + "INNER JOIN (SELECT 1 number) n ON n.number = c.number"; + try (Connection conn = getConnection(url, props); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery(sql)) { + String firstColumn = rs.getMetaData().getColumnName(1); + String secondColumn = rs.getMetaData().getColumnName(2); + + if (!rs.next()) { + throw new IllegalStateException("Should have at least one record"); + } + + // return "stringValue=string, number=1" + // instead of "simple.string_value=string, n.number=1", + return String.format("%s=%s, %s=%s", firstColumn, rs.getString(1), secondColumn, rs.getInt(2)); + } + } + + static String unwrapToUseClientApi(String url) throws SQLException { + String sql = "select 1 n union all select 2 n"; + try (Connection conn = getConnection(url); Statement stmt = conn.createStatement()) { + ClickHouseRequest request = stmt.unwrap(ClickHouseRequest.class); + // server setting is not allowed in read-only mode + if (!conn.isReadOnly()) { + // not required for ClickHouse 22.7+, only works for HTTP protocol + request.set("send_progress_in_http_headers", 1); + } + try (ClickHouseResponse response = request.query(sql).executeAndWait()) { + int count = 0; + // may throw UncheckedIOException (due to restriction of Iterable interface) + for (ClickHouseRecord r : response.records()) { + count++; + } + return String.format("Result Rows: %d (read bytes: %d)", count, response.getSummary().getReadBytes()); + } catch (ClickHouseException e) { + throw SqlExceptionUtils.handle(e); + } + } + } + + public static void main(String[] args) { + // randomly pick one of the two endpoints to connect to, + // fail over to the other when there's connection issue + String url = System.getProperty("chUrl", + "jdbc:ch://(https://explorer@play.clickhouse.com:443)," + + "(https://demo:demo@github.demo.trial.altinity.cloud)" + + "/default?failover=1&load_balancing_policy=random"); + + try { + System.out.println(exteralTables(url)); + System.out.println(namedParameter(url)); + System.out.println(renameResponseColumn(url)); + System.out.println(unwrapToUseClientApi(url)); + + // requires ClickHouse 22.6+ with transaction enabled + System.out.println(manualTransaction(url)); + } catch (SQLException e) { + e.printStackTrace(); + } + } +} diff --git a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java index c25383c80..a0cc834b9 100644 --- a/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java +++ b/examples/jdbc/src/main/java/com/clickhouse/examples/jdbc/Basic.java @@ -6,83 +6,147 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.time.LocalDateTime; +import java.util.Properties; public class Basic { - static void dropAndCreateTable(String url, String user, String password, String table) throws SQLException { - try (Connection conn = DriverManager.getConnection(url, user, password); - Statement stmt = conn.createStatement()) { - stmt.execute( - String.format( - "drop table if exists %1$s; create table %1$s(a String, b Nullable(String)) engine=Memory", - table)); + static final String TABLE_NAME = "jdbc_example_basic"; + + private static Connection getConnection(String url) throws SQLException { + return getConnection(url, new Properties()); + } + + private static Connection getConnection(String url, Properties properties) throws SQLException { + final Connection conn; + // Driver driver = new ClickHouseDriver(); + // conn = driver.connect(url, properties); + + // ClickHouseDataSource dataSource = new ClickHouseDataSource(url, properties); + // conn = dataSource.getConnection(); + + conn = DriverManager.getConnection(url, properties); + System.out.println("Connected to: " + conn.getMetaData().getURL()); + return conn; + } + + static int dropAndCreateTable(Connection conn) throws SQLException { + try (Statement stmt = conn.createStatement()) { + // multi-statement query is supported by default + // session will be created automatically during execution + stmt.execute(String.format( + "drop table if exists %1$s; create table %1$s(a String, b Nullable(String)) engine=Memory", + TABLE_NAME)); + return stmt.getUpdateCount(); } } - static void batchInsert(String url, String user, String password, String table) throws SQLException { - try (Connection conn = DriverManager.getConnection(url, user, password)) { - // not that fast as it's based on string substitution and large sql statement - String sql = String.format("insert into %1$s values(?, ?)", table); - try (PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setString(1, "a"); - ps.setString(2, "b"); - ps.addBatch(); - ps.setString(1, "c"); - ps.setString(2, null); - ps.addBatch(); - ps.executeBatch(); + static int batchInsert(Connection conn) throws SQLException { + // 1. NOT recommended when inserting lots of rows, because it's based on a large statement + String sql = String.format("insert into %s values(? || ' - 1', ?)", TABLE_NAME); + int count = 0; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, "a"); + ps.setString(2, "b"); + ps.addBatch(); + ps.setString(1, "c"); + // ps.setNull(2, Types.VARCHAR); + // ps.setObject(2, null); + ps.setString(2, null); + ps.addBatch(); + // same as below query: + // insert into values ('a' || ' - 1', 'b'), ('c' || ' - 1', null) + for (int i : ps.executeBatch()) { + if (i > 0) { + count += i; + } } + } - // faster when inserting massive data - sql = String.format("insert into %1$s", table); - try (PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setString(1, "a"); - ps.setString(2, "b"); - ps.addBatch(); - ps.setString(1, "c"); - ps.setString(2, null); - ps.addBatch(); - ps.executeBatch(); + // 2. faster and ease of use, with additional query for getting table structure + // sql = String.format("insert into %s (a)", TABLE_NAME); + // sql = String.format("insert into %s (a) values (?)", TABLE_NAME); + sql = String.format("insert into %s (* except b)", TABLE_NAME); + // Note: below query will be issued to get table structure: + // select * except b from
where 0 + try (PreparedStatement ps = conn.prepareStatement(sql)) { + // implicit type conversion: int -> String + ps.setInt(1, 1); + ps.addBatch(); + // implicit type conversion: LocalDateTime -> string + ps.setObject(1, LocalDateTime.now()); + ps.addBatch(); + // same as below query: + // insert into
format RowBinary + for (int i : ps.executeBatch()) { + if (i > 0) { + count += i; + } } + } - // fastest approach as it does not need to issue additional query for metadata - sql = String.format("insert into %1$s select a, b from input('a String, b Nullable(String)')", table); - try (PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setString(1, "a"); - ps.setString(2, "b"); - ps.addBatch(); - ps.setString(1, "c"); - ps.setString(2, null); - ps.addBatch(); - ps.executeBatch(); + // 3. fastest but inconvenient and NOT portable(as it's limited to ClickHouse) + // see https://clickhouse.com/docs/en/sql-reference/table-functions/input/ + sql = String.format("insert into %s select a, b from input('a String, b Nullable(String)')", TABLE_NAME); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, "a"); + ps.setString(2, "b"); + ps.addBatch(); + ps.setString(1, "c"); + ps.setString(2, null); + ps.addBatch(); + // same as below query: + // insert into
format RowBinary + for (int i : ps.executeBatch()) { + if (i > 0) { + count += i; + } } } + + return count; } - static int query(String url, String user, String password, String table) throws SQLException { - try (Connection conn = DriverManager.getConnection(url, user, password); + static int connectWithCustomSettings(String url) throws SQLException { + // comma separated settings + String customSettings = "session_check=0,max_query_size=100"; + Properties properties = new Properties(); + // properties.setProperty(ClickHouseHttpClientOption.CUSTOM_PARAMS.getKey(), + // customSettings); + properties.setProperty("custom_http_params", customSettings); // limited to http protocol + try (Connection conn = getConnection(url, properties); Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("select * from " + table)) { + ResultSet rs = stmt.executeQuery("select 5")) { + return rs.next() ? rs.getInt(1) : -1; + } + } + + static int query(Connection conn) throws SQLException { + String sql = "select * from " + TABLE_NAME; + try (Statement stmt = conn.createStatement()) { + // set max_result_rows = 3, result_overflow_mode = 'break' + // or simply discard rows after the first 3 in read-only mode + stmt.setMaxRows(3); int count = 0; - while (rs.next()) { - count++; + try (ResultSet rs = stmt.executeQuery(sql)) { + while (rs.next()) { + count++; + } } return count; } } public static void main(String[] args) { - String url = String.format("jdbc:ch://%s:%d/system", System.getProperty("chHost", "localhost"), - Integer.parseInt(System.getProperty("chPort", "8123"))); - String user = System.getProperty("chUser", "default"); - String password = System.getProperty("chPassword", ""); - String table = "jdbc_example_basic"; - - try { - dropAndCreateTable(url, user, password, table); + // jdbc:ch:https://explorer@play.clickhouse.com:443 + // jdbc:ch:https://demo:demo@github.demo.trial.altinity.cloud + String url = System.getProperty("chUrl", "jdbc:ch://localhost"); - batchInsert(url, user, password, table); + try (Connection conn = getConnection(url)) { + connectWithCustomSettings(url); - query(url, user, password, table); + System.out.println("Update Count: " + dropAndCreateTable(conn)); + System.out.println("Inserted Rows: " + batchInsert(conn)); + System.out.println("Result Rows: " + query(conn)); } catch (SQLException e) { e.printStackTrace(); }