Skip to content

Commit

Permalink
Merge pull request #783 from zhicwu/enhance-jdbc
Browse files Browse the repository at this point in the history
Prepare 0.3.2
  • Loading branch information
zhicwu committed Dec 29, 2021
2 parents 48a6281 + 82868de commit 08058fe
Show file tree
Hide file tree
Showing 67 changed files with 3,030 additions and 922 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/benchmark.yml
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
mvn --batch-mode --update-snapshots -q -DskipTests install
cd clickhouse-benchmark
java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf text \
-p client=clickhouse-http-jdbc -p client=clickhouse-grpc-jdbc -p type=default Basic
-p client=clickhouse-http-jdbc1 -p client=clickhouse-grpc-jdbc -p type=object Basic
echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
cat jmh-result.text >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
Expand Down
301 changes: 150 additions & 151 deletions README.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ public void doSetup(ServerState serverState) throws Exception {
client = createClient();

String[] sqls = new String[] { "drop table if exists system.test_insert",
"create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory" };
"create table if not exists system.test_insert(id String, i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory" };

for (String sql : sqls) {
try (ClickHouseResponse resp = client.connect(server).query(sql).execute().get()) {
Expand All @@ -98,10 +98,13 @@ public void doSetup(ServerState serverState) throws Exception {
}

@TearDown(Level.Trial)
public void doTearDown(ServerState serverState) {
public void doTearDown(ServerState serverState) throws Exception {
dispose();

if (client != null) {
try (ClickHouseResponse resp = client.connect(server).query("truncate table system.test_insert").execute()
.get()) {

} finally {
try {
client.close();
} catch (Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ public class Basic extends DriverBenchmark {
public int insertOneRandomNumber(DriverState state) throws Throwable {
final int num = state.getRandomSample();

return executeInsert(state, "insert into test_insert(i) values(?)",
return executeInsert(state, "insert into test_insert(i) values(?)", (p, v, l, i) -> p.setObject(i, v),
Collections.enumeration(Collections.singletonList(new Object[] { num })));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,11 @@ public abstract class DriverBenchmark {
private final int BATCH_SIZE = Integer.parseInt(System.getProperty("batchSize", "5000"));
// fetch size for query
private final int FETCH_SIZE = Integer.parseInt(System.getProperty("fetchSize", "1000"));
// insert mode: 1) values; 2) table; 3) input
private final String INSERT_MODE = System.getProperty("insertMode", "values").toLowerCase();

protected PreparedStatement setParameters(PreparedStatement s, Object... values) throws SQLException {
protected PreparedStatement setParameters(PreparedStatement s, Object... values)
throws SQLException {
if (values != null && values.length > 0) {
int index = 1;
for (Object v : values) {
Expand Down Expand Up @@ -66,14 +69,22 @@ protected String replaceParameters(String sql, Object... values) {
return sql;
}

private int processBatch(Statement s, String sql, Enumeration<Object[]> generator) throws SQLException {
private int processBatch(Statement s, String sql, SupplyValueFunction func, Enumeration<Object[]> generator)
throws SQLException {
int rows = 0;
int counter = 0;
PreparedStatement ps = s instanceof PreparedStatement ? (PreparedStatement) s : null;
while (generator.hasMoreElements()) {
Object[] values = generator.nextElement();
if (ps != null) {
setParameters(ps, values);
int colIndex = 1;
for (Object v : values) {
if (colIndex == 1 && v instanceof String) {
ps.setString(colIndex++, (String) v);
} else {
func.set(ps, v, rows, colIndex++);
}
}

if (BATCH_SIZE > 0) {
ps.addBatch();
Expand Down Expand Up @@ -103,19 +114,26 @@ private int processBatch(Statement s, String sql, Enumeration<Object[]> generato
return rows;
}

protected int executeInsert(DriverState state, String sql, Enumeration<Object[]> generator) throws SQLException {
protected int executeInsert(DriverState state, String sql, SupplyValueFunction func,
Enumeration<Object[]> generator) throws SQLException {
Objects.requireNonNull(generator);

final Connection conn = state.getConnection();
int rows = 0;

if ("table".equals(INSERT_MODE)) {
sql = sql.substring(0, sql.indexOf('\n') + 1);
} else if ("input".equals(INSERT_MODE)) {
sql = sql.substring(0, sql.indexOf('\n')).replaceFirst("--", "");
}

if (state.usePreparedStatement()) {
try (PreparedStatement s = conn.prepareStatement(sql)) {
rows = processBatch(s, sql, generator);
rows = processBatch(s, sql, func, generator);
}
} else {
try (Statement s = conn.createStatement()) {
rows = processBatch(s, sql, generator);
rows = processBatch(s, sql, func, generator);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import java.util.UUID;

import org.openjdk.jmh.annotations.Level;
import org.openjdk.jmh.annotations.Param;
Expand Down Expand Up @@ -44,20 +45,22 @@ public void doSetup(ServerState serverState) throws Exception {
JdbcDriver jdbcDriver = JdbcDriver.from(client);

String compression = String.valueOf(Boolean.parseBoolean(System.getProperty("compression", "true")));
String additional = System.getProperty("additional", "");

try {
driver = (java.sql.Driver) Class.forName(jdbcDriver.getClassName()).getDeclaredConstructor().newInstance();
url = String.format(jdbcDriver.getUrlTemplate(), serverState.getHost(),
serverState.getPort(jdbcDriver.getDefaultPort()), serverState.getDatabase(), serverState.getUser(),
serverState.getPassword(), compression);
serverState.getPassword(), compression, additional);
// ClickHouseDefines.WRITE_COMPRESS = false;
// ClickHouseDefines.READ_DECOMPRESS = Boolean.parseBoolean(compression);
conn = driver.connect(url, new Properties());

try (Statement s = conn.createStatement()) {
// s.execute("drop table if exists system.test_insert");
s.execute("truncate table if exists system.test_insert");
s.execute(
"create table if not exists system.test_insert(i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory");
"create table if not exists system.test_insert(b String, i Nullable(UInt64), s Nullable(String), t Nullable(DateTime))engine=Memory");
}

if (!Constants.REUSE_CONNECTION.equalsIgnoreCase(connection)) {
Expand Down Expand Up @@ -138,4 +141,16 @@ public ConsumeValueFunction getConsumeFunction(ConsumeValueFunction defaultFunc)
return defaultFunc;
}
}

public SupplyValueFunction getSupplyFunction(SupplyValueFunction defaultFunc) {
if ("string".equals(type)) {
return (p, v, l, i) -> p.setString(i, v != null ? v.toString() : null);
} else if ("object".equals(type)) {
return (p, v, l, i) -> p.setObject(i, v);
} else if (defaultFunc == null) {
return (p, v, l, i) -> p.setObject(i, v);
} else {
return defaultFunc;
}
}
}
Original file line number Diff line number Diff line change
@@ -1,67 +1,106 @@
package com.clickhouse.benchmark.jdbc;

import java.sql.ResultSet;
import java.sql.Statement;
import java.sql.Timestamp;
import java.time.LocalDateTime;
import java.time.ZoneOffset;
import java.util.Enumeration;
import java.util.Locale;
import java.util.UUID;

import org.openjdk.jmh.annotations.Benchmark;

public class Insertion extends DriverBenchmark {
private void checkResult(DriverState state, String batchId, int expectedRows, int actualResult) throws Exception {
boolean isValid = actualResult == expectedRows;
if (isValid) {
try (Statement stmt = executeQuery(state,
"select toInt32(count(1)) from system.test_insert where b=?", batchId)) {
ResultSet rs = stmt.getResultSet();
isValid = rs.next() && (actualResult = rs.getInt(1)) == expectedRows;
}
}
if (!isValid) {
throw new IllegalStateException(String.format(Locale.ROOT,
"Expected %d rows being inserted but we got %d", expectedRows, actualResult));
}
}

@Benchmark
public int insert10kUInt64Rows(DriverState state) throws Throwable {
public void insertInt64(DriverState state) throws Throwable {
final int range = state.getRandomNumber();
final int rows = state.getSampleSize() + range;
final String batchId = UUID.randomUUID().toString();
SupplyValueFunction func = state.getSupplyFunction((p, v, l, i) -> p.setLong(i, (long) v));
int result = executeInsert(state,
"insert into system.test_insert(b,i) -- select b,v from input('b String, v Int64')\nvalues(?,?)", func,
new Enumeration<Object[]>() {
int counter = 0;

return executeInsert(state, "insert into system.test_insert(i) values(?)", new Enumeration<Object[]>() {
int counter = 0;
@Override
public boolean hasMoreElements() {
return counter < rows;
}

@Override
public boolean hasMoreElements() {
return counter < rows;
}
@Override
public Object[] nextElement() {
return new Object[] { batchId, (long) (range + (counter++)) };
}
});

@Override
public Object[] nextElement() {
return new Object[] { range + (counter++) };
}
});
checkResult(state, batchId, rows, result);
}

@Benchmark
public int insert10kStringRows(DriverState state) throws Throwable {
public void insertString(DriverState state) throws Throwable {
final int range = state.getRandomNumber();
final int rows = state.getSampleSize() + range;
final String batchId = UUID.randomUUID().toString();
SupplyValueFunction func = state.getSupplyFunction((p, v, l, i) -> p.setString(i, (String) v));
int result = executeInsert(state,
"insert into system.test_insert(b, s) -- select b, v from input('b String, v String')\nvalues(?, ?)",
func, new Enumeration<Object[]>() {
int counter = 0;

return executeInsert(state, "insert into system.test_insert(s) values(?)", new Enumeration<Object[]>() {
int counter = 0;
@Override
public boolean hasMoreElements() {
return counter < rows;
}

@Override
public boolean hasMoreElements() {
return counter < rows;
}
@Override
public Object[] nextElement() {
return new Object[] { batchId, String.valueOf(range + (counter++)) };
}
});

@Override
public Object[] nextElement() {
return new Object[] { String.valueOf(range + (counter++)) };
}
});
checkResult(state, batchId, rows, result);
}

@Benchmark
public int insert10kTimestampRows(DriverState state) throws Throwable {
public void insertTimestamp(DriverState state) throws Throwable {
final int range = state.getRandomNumber();
final int rows = state.getSampleSize() + range;
final String batchId = UUID.randomUUID().toString();
SupplyValueFunction func = state
.getSupplyFunction((p, v, l, i) -> p.setTimestamp(i, Timestamp.valueOf((LocalDateTime) v)));
int result = executeInsert(state,
"insert into system.test_insert(b,t) -- select b,v from input('b String,v DateTime32')\nvalues(?,?)",
func, new Enumeration<Object[]>() {
int counter = 0;

return executeInsert(state, "insert into system.test_insert(t) values(?)", new Enumeration<Object[]>() {
int counter = 0;
@Override
public boolean hasMoreElements() {
return counter < rows;
}

@Override
public boolean hasMoreElements() {
return counter < rows;
}
@Override
public Object[] nextElement() {
return new Object[] { batchId,
LocalDateTime.ofEpochSecond((long) range + (counter++), 0, ZoneOffset.UTC) };
}
});

@Override
public Object[] nextElement() {
return new Object[] { new Timestamp((long) range + (counter++)) };
}
});
checkResult(state, batchId, rows, result);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,41 @@
public enum JdbcDriver {
// ClickHouse4j
Clickhouse4j("cc.blynk.clickhouse.ClickHouseDriver",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
Constants.HTTP_PORT),
// ClickHouse JDBC Driver
ClickhouseHttpJdbc1("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_URL_CONNECTION&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
Constants.HTTP_PORT),
ClickhouseHttpJdbc2("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
"jdbc:ch://%s:%s/%s?http_connection_provider=HTTP_CLIENT&ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
Constants.HTTP_PORT),
ClickhouseGrpcJdbc("com.clickhouse.jdbc.ClickHouseDriver",
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s",
"jdbc:ch:grpc://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&max_inbound_message_size=2147483647&compress=%s%s",
Constants.GRPC_PORT),
ClickhouseJdbc("ru.yandex.clickhouse.ClickHouseDriver",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
Constants.HTTP_PORT),
// ClickHouse Native JDBC Driver
ClickhouseNativeJdbc("com.github.housepower.jdbc.ClickHouseDriver",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s",
"jdbc:clickhouse://%s:%s/%s?ssl=false&user=%s&password=%s&use_server_time_zone=false&use_time_zone=UTC&compress=%s%s",
Constants.NATIVE_PORT),

// MariaDB Java Client
MariadbJavaClient("org.mariadb.jdbc.Driver",
"jdbc:mariadb://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false&useCompression=%s"
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC",
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&serverTimezone=UTC%s",
Constants.MYSQL_PORT),

// MySQL Connector/J
MysqlConnectorJava("com.mysql.cj.jdbc.Driver",
"jdbc:mysql://%s:%s/%s?user=%s&password=%s&useSSL=false&useServerPrepStmts=false"
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC&useCompression=%s",
+ "&rewriteBatchedStatements=true&cachePrepStmts=true&connectionTimeZone=UTC&useCompression=%S%s",
Constants.MYSQL_PORT),

// PostgreSQL JDBC Driver
PostgresqlJdbc("org.postgresql.Driver",
"jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple&compress=%s",
"jdbc:postgresql://%s:%s/%s?user=%s&password=%s&ssl=false&sslmode=disable&preferQueryMode=simple&compress=%s%s",
Constants.POSTGRESQL_PORT);

private final String className;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
package com.clickhouse.benchmark.jdbc;

import java.sql.PreparedStatement;
import java.sql.SQLException;

@FunctionalInterface
public interface SupplyValueFunction {
void set(PreparedStatement ps, Object value, int rowIndex, int columnIndex) throws SQLException;
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import com.clickhouse.benchmark.BaseState;
import com.clickhouse.client.ClickHouseEnum;
import com.clickhouse.client.ClickHouseValue;
import com.clickhouse.client.ClickHouseValues;
import com.clickhouse.client.data.ClickHouseBigDecimalValue;
import com.clickhouse.client.data.ClickHouseBigIntegerValue;
import com.clickhouse.client.data.ClickHouseByteValue;
Expand Down Expand Up @@ -89,8 +90,8 @@ public void setupSamples() {

// add(map, list, Object[].class, () -> ClickHouseArrayValue.of((Object[]) o));
add(map, list, LocalDate.class, () -> ClickHouseDateValue.ofNull());
add(map, list, LocalTime.class, () -> ClickHouseDateTimeValue.ofNull(0));
add(map, list, LocalDateTime.class, () -> ClickHouseDateTimeValue.ofNull(0));
add(map, list, LocalTime.class, () -> ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE));
add(map, list, LocalDateTime.class, () -> ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE));
add(map, list, String.class, () -> ClickHouseStringValue.ofNull());

mappings = Collections.unmodifiableMap(map);
Expand Down Expand Up @@ -130,9 +131,9 @@ ClickHouseValue newValue(Class<?> clazz) {
} else if (LocalDate.class.equals(clazz)) {
return ClickHouseDateValue.ofNull();
} else if (LocalTime.class.equals(clazz)) {
return ClickHouseDateTimeValue.ofNull(0);
return ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE);
} else if (LocalDateTime.class.equals(clazz)) {
return ClickHouseDateTimeValue.ofNull(0);
return ClickHouseDateTimeValue.ofNull(0, ClickHouseValues.UTC_TIMEZONE);
} else if (String.class.equals(clazz)) {
return ClickHouseStringValue.ofNull();
}
Expand Down
Loading

0 comments on commit 08058fe

Please sign in to comment.