Skip to content

Enable lightweight delete support and fix minor issues in sight #1063

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 1, 2022
Merged

Enable lightweight delete support and fix minor issues in sight #1063

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 95 additions & 8 deletions clickhouse-client/src/main/java/com/clickhouse/client/ClickHouseRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -413,13 +413,15 @@ protected ClickHouseRequest(ClickHouseClient client, Function<ClickHouseNodeSele
this.server = (Function<ClickHouseNodeSelector, ClickHouseNode> & Serializable) server;
this.serverRef = ref == null ? new AtomicReference<>(null) : ref;
this.txRef = new AtomicReference<>(null);
this.sealed = sealed;

this.externalTables = new LinkedList<>();
this.options = options != null ? new HashMap<>(options) : new HashMap<>();
this.options = new HashMap<>();
this.settings = new LinkedHashMap<>(client.getConfig().getCustomSettings());
options(options);

this.namedParameters = new HashMap<>();

this.sealed = sealed;
}

protected <T> T changeProperty(String property, T oldValue, T newValue) {
Expand Down Expand Up @@ -673,6 +675,47 @@ public ClickHouseParameterizedQuery getPreparedQuery() {
return preparedQuery;
}

/**
* Gets typed setting value.
*
* @param <T> type of the setting value
* @param setting non-null setting key
* @param valueType non-null value type
* @return non-null value
*/
public <T extends Serializable> T getSetting(String setting, Class<T> valueType) {
Serializable value = settings.get(ClickHouseChecker.nonBlank(setting, PARAM_SETTING));
return ClickHouseOption.fromString(value == null ? "" : value.toString(), valueType);
}

/**
* Gets typed setting value.
*
* @param <T> type of the setting value
* @param setting non-null setting key
* @param defaultValue non-null default value
* @return non-null value
*/
public <T extends Serializable> T getSetting(String setting, T defaultValue) {
Serializable value = settings.get(ClickHouseChecker.nonBlank(setting, PARAM_SETTING));
ClickHouseChecker.nonNull(defaultValue, "defaultValue");
if (value == null) {
return defaultValue;
}

return (T) ClickHouseOption.fromString(value.toString(), defaultValue.getClass());
}

/**
* Checks if a setting has been defined or not.
*
* @param setting setting
* @return true if the setting has been defined; false otherwise
*/
public boolean hasSetting(String setting) {
return settings.containsKey(setting);
}

/**
* Gets immutable settings.
*
Expand Down Expand Up @@ -1087,6 +1130,26 @@ public SelfT options(Properties options) {
return (SelfT) this;
}

/**
* Checks if a option has been defined or not.
*
* @param option option
* @return true if the option has been defined; false otherwise
*/
public boolean hasOption(ClickHouseOption option) {
return options.containsKey(option);
}

/**
* Checks if a option has been defined or not.
*
* @param key key of the option
* @return true if the option has been defined; false otherwise
*/
public boolean hasOption(String key) {
return options.containsKey(ClickHouseClientOption.fromKey(key));
}

/**
* Sets output file, to which response will be redirected.
*
Expand Down Expand Up @@ -1503,7 +1566,8 @@ public SelfT query(String query, String queryId) {
}

/**
* Sets all server settings.
* Sets all server settings. When {@code settings} is null or empty, it's same
* as {@link #clearSettings()}.
*
* @param settings settings
* @return the request itself
Expand All @@ -1523,7 +1587,7 @@ public SelfT settings(Map<String, Serializable> settings) {
} else {
Map<String, Serializable> m = new HashMap<>();
m.putAll(this.settings);
if (options != null) {
if (settings != null) {
for (Entry<String, Serializable> e : settings.entrySet()) {
set(e.getKey(), e.getValue());
m.remove(e.getKey());
Expand All @@ -1537,6 +1601,29 @@ public SelfT settings(Map<String, Serializable> settings) {
return (SelfT) this;
}

/**
* Clears server settings.
*
* @return the request itself
*/
@SuppressWarnings("unchecked")
public SelfT clearSettings() {
checkSealed();

if (!this.settings.isEmpty()) {
if (changeListener == null) {
this.settings.clear();
resetCache();
} else {
for (Iterator<String> it = settings.keySet().iterator(); it.hasNext();) {
removeSetting(it.next());
}
}
}

return (SelfT) this;
}

/**
* Clears session configuration including session id, session check(whether to
* validate the id), and session timeout. Transaction will be removed as well.
Expand Down Expand Up @@ -1911,11 +1998,11 @@ public SelfT reset() {
this.options.clear();
this.settings.clear();
} else {
for (ClickHouseOption o : this.options.keySet().toArray(new ClickHouseOption[0])) {
removeOption(o);
for (Iterator<ClickHouseOption> it = this.options.keySet().iterator(); it.hasNext();) {
removeOption(it.next());
}
for (String s : this.settings.keySet().toArray(new String[0])) {
removeSetting(s);
for (Iterator<String> it = this.settings.keySet().iterator(); it.hasNext();) {
removeSetting(it.next());
}
}
this.input = changeProperty(PROP_DATA, this.input, null);
Expand Down
51 changes: 39 additions & 12 deletions clickhouse-client/src/main/java/com/clickhouse/client/config/ClickHouseOption.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
import java.util.Optional;
import java.util.TimeZone;

import com.clickhouse.client.ClickHouseChecker;

/**
* This defines a configuration option. To put it in a nutshell, an option is
* composed of key, default value(which implies type of the value) and
Expand Down Expand Up @@ -71,35 +73,39 @@ public static Map<String, String> toKeyValuePairs(String str) {
* @param <T> type of the value
* @param value value in string format
* @param clazz non-null class of the value
* @return typed value
* @return non-null typed value
*/
@SuppressWarnings("unchecked")
static <T extends Serializable> T fromString(String value, Class<T> clazz) {
if (value == null || clazz == null) {
throw new IllegalArgumentException("Non-null value and class are required");
public static <T extends Serializable> T fromString(String value, Class<T> clazz) {
if (clazz == null) {
throw new IllegalArgumentException("Non-null value type is required");
} else if (value == null) {
value = "";
}

T result;
if (clazz == boolean.class || clazz == Boolean.class) {
final Boolean boolValue;
if ("1".equals(value) || "0".equals(value)) {
if (value.isEmpty()) {
boolValue = Boolean.FALSE;
} else if (value.length() == 1) {
boolValue = "1".equals(value);
} else {
boolValue = Boolean.valueOf(value);
}
result = clazz.cast(boolValue);
result = (T) boolValue;
} else if (byte.class == clazz || Byte.class == clazz) {
result = clazz.cast(value.isEmpty() ? Byte.valueOf((byte) 0) : Byte.valueOf(value));
result = (T) (value.isEmpty() ? Byte.valueOf((byte) 0) : Byte.valueOf(value));
} else if (short.class == clazz || Short.class == clazz) {
result = clazz.cast(value.isEmpty() ? Short.valueOf((short) 0) : Short.valueOf(value));
result = (T) (value.isEmpty() ? Short.valueOf((short) 0) : Short.valueOf(value));
} else if (int.class == clazz || Integer.class == clazz) {
result = clazz.cast(value.isEmpty() ? Integer.valueOf(0) : Integer.valueOf(value));
result = (T) (value.isEmpty() ? Integer.valueOf(0) : Integer.valueOf(value));
} else if (long.class == clazz || Long.class == clazz) {
result = clazz.cast(value.isEmpty() ? Long.valueOf(0L) : Long.valueOf(value));
result = (T) (value.isEmpty() ? Long.valueOf(0L) : Long.valueOf(value));
} else if (float.class == clazz || Float.class == clazz) {
result = clazz.cast(value.isEmpty() ? Float.valueOf(0F) : Float.valueOf(value));
result = (T) (value.isEmpty() ? Float.valueOf(0F) : Float.valueOf(value));
} else if (double.class == clazz || Double.class == clazz) {
result = clazz.cast(value.isEmpty() ? Double.valueOf(0D) : Double.valueOf(value));
result = (T) (value.isEmpty() ? Double.valueOf(0D) : Double.valueOf(value));
} else if (Enum.class.isAssignableFrom(clazz)) {
Enum enumValue = null;
try {
Expand Down Expand Up @@ -127,6 +133,27 @@ static <T extends Serializable> T fromString(String value, Class<T> clazz) {
return result;
}

/**
* Converts given string to typed value. When {@code value} is null or blank,
* {@code defaultValue} will be returned.
*
* @param <T> type of the value
* @param value value in string format
* @param defaultValue non-null default value
* @return non-null typed value
*/
@SuppressWarnings("unchecked")
public static <T extends Serializable> T fromString(String value, T defaultValue) {
if (defaultValue == null) {
throw new IllegalArgumentException("Non-null default value is required");
}
if (ClickHouseChecker.isNullOrBlank(value)) {
return defaultValue;
}

return (T) fromString(value, defaultValue.getClass());
}

/**
* Gets default value of the option.
*
Expand Down
56 changes: 43 additions & 13 deletions clickhouse-client/src/test/java/com/clickhouse/client/ClickHouseRequestTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,11 +116,14 @@ public void settingChanged(ClickHouseRequest<?> source, String setting, Serializ
Assert.assertEquals(changedOptions.toArray(new Object[0]),
new Object[][] {
new Object[] { request, ClickHouseClientOption.ASYNC, null, false },
new Object[] { request, ClickHouseClientOption.FORMAT, null, ClickHouseFormat.Arrow },
new Object[] { request, ClickHouseClientOption.FORMAT, ClickHouseFormat.Arrow,
new Object[] { request, ClickHouseClientOption.FORMAT, null,
ClickHouseFormat.Arrow },
new Object[] { request, ClickHouseClientOption.FORMAT,
ClickHouseFormat.Arrow,
ClickHouseFormat.Avro },
new Object[] { request, ClickHouseClientOption.ASYNC, false, null },
new Object[] { request, ClickHouseClientOption.FORMAT, ClickHouseFormat.Avro, null } });
new Object[] { request, ClickHouseClientOption.FORMAT,
ClickHouseFormat.Avro, null } });
Assert.assertEquals(changedProperties.toArray(new Object[0]), new Object[][] {
{ request, ClickHouseRequest.PROP_QUERY, null, "select 1" },
{ request, ClickHouseRequest.PROP_QUERY, "select 1", "select 2" },
Expand Down Expand Up @@ -160,15 +163,18 @@ public void testServerListener() {
ClickHouseRequest<?> request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build());
final List<Object[]> serverChanges = new ArrayList<>();
request.setServerListener(
(currentServer, newServer) -> serverChanges.add(new Object[] { currentServer, newServer }));
(currentServer, newServer) -> serverChanges
.add(new Object[] { currentServer, newServer }));
ClickHouseNode s11 = ClickHouseNode.of("http://node1");
ClickHouseNode s12 = ClickHouseNode.of("grpc://node1/system");
ClickHouseNode s21 = ClickHouseNode.of("tcp://node2");
ClickHouseNode s22 = ClickHouseNode.of("https://node2");
request.changeServer(request.getServer(), s11);
Assert.assertEquals(serverChanges.toArray(new Object[0]), new Object[][] { { ClickHouseNode.DEFAULT, s11 } });
Assert.assertEquals(serverChanges.toArray(new Object[0]),
new Object[][] { { ClickHouseNode.DEFAULT, s11 } });
request.changeServer(ClickHouseNode.DEFAULT, s12);
Assert.assertEquals(serverChanges.toArray(new Object[0]), new Object[][] { { ClickHouseNode.DEFAULT, s11 } });
Assert.assertEquals(serverChanges.toArray(new Object[0]),
new Object[][] { { ClickHouseNode.DEFAULT, s11 } });
request.changeServer(s11, s21);
Assert.assertEquals(serverChanges.toArray(new Object[0]),
new Object[][] { { ClickHouseNode.DEFAULT, s11 }, { s11, s21 } });
Expand All @@ -183,7 +189,8 @@ public void testCopy() {
ClickHouseRequest<?> request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build());
request.compressServerResponse(true, ClickHouseCompression.BROTLI, 2);
request.decompressClientRequest(true, ClickHouseCompression.ZSTD, 5);
request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0])).build());
request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0]))
.build());
request.format(ClickHouseFormat.Avro);
request.table("table1", "query_id1");
request.query("select :a", UUID.randomUUID().toString());
Expand Down Expand Up @@ -244,12 +251,30 @@ public void testFormat() {
Assert.assertEquals(request.getFormat(),
(ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue());
Assert.assertEquals(request.getInputFormat(),
((ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue()).defaultInputFormat());
((ClickHouseFormat) ClickHouseDefaults.FORMAT.getEffectiveDefaultValue())
.defaultInputFormat());
request.format(ClickHouseFormat.Arrow);
Assert.assertEquals(request.getFormat(), ClickHouseFormat.Arrow);
Assert.assertEquals(request.getInputFormat(), ClickHouseFormat.Arrow);
}

@Test(groups = { "unit" })
public void testGetSetting() {
ClickHouseRequest<?> request = ClickHouseClient.newInstance()
.connect("http://localhost?custom_settings=a%3D1%2Cb%3D2");
Assert.assertEquals(request.getSetting("a", boolean.class), true);
Assert.assertEquals(request.getSetting("a", Boolean.class), true);
Assert.assertEquals(request.getSetting("a", false), true);
Assert.assertEquals(request.getSetting("a", int.class), 1);
Assert.assertEquals(request.getSetting("a", Integer.class), 1);
Assert.assertEquals(request.getSetting("a", 9), 1);
Assert.assertEquals(request.getSetting("b", "3"), "2");
// request.settings(null);
request.clearSettings();
Assert.assertTrue(request.getSettings().isEmpty());
Assert.assertEquals(request.getSetting("b", 9), 9);
}

@Test(groups = { "unit" })
public void testNamedParameters() {
// String sql = "select xxx from xxx settings max_execution_time =
Expand Down Expand Up @@ -293,7 +318,8 @@ public void testParams() {
"select -128 as one, NULL as two, * from my_table where key=NULL and arr[NULL] in numbers(NULL)");

request.params(ClickHouseStringValue.of(""),
ClickHouseDateTimeValue.of("2012-12-12 12:23:34.56789", 2, ClickHouseValues.UTC_TIMEZONE),
ClickHouseDateTimeValue.of("2012-12-12 12:23:34.56789", 2,
ClickHouseValues.UTC_TIMEZONE),
ClickHouseStringValue.of("key"), ClickHouseIntegerValue.of(1),
ClickHouseBigIntegerValue.of(BigInteger.TEN));
Assert.assertEquals(request.getQuery(), sql);
Expand All @@ -310,8 +336,10 @@ public void testParams() {
"select 1.0 as one, NULL as two, * from my_table where key=NULL and arr[NULL] in numbers(NULL)");

params.put("one", ClickHouseStringValue.of("").toSqlExpression());
params.put("two", ClickHouseDateTimeValue.of("2012-12-12 12:23:34.56789", 2, ClickHouseValues.UTC_TIMEZONE)
.toSqlExpression());
params.put("two",
ClickHouseDateTimeValue
.of("2012-12-12 12:23:34.56789", 2, ClickHouseValues.UTC_TIMEZONE)
.toSqlExpression());
params.put("key", ClickHouseStringValue.of("key").toSqlExpression());
params.put("some", ClickHouseBigIntegerValue.of(BigInteger.ONE).toSqlExpression());
params.put("idx", ClickHouseIntegerValue.of(1).toSqlExpression());
Expand All @@ -328,7 +356,8 @@ public void testSeal() {
ClickHouseRequest<?> request = ClickHouseClient.newInstance().connect(ClickHouseNode.builder().build());
request.compressServerResponse(true, ClickHouseCompression.BROTLI, 2);
request.decompressClientRequest(true, ClickHouseCompression.ZSTD, 5);
request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0])).build());
request.external(ClickHouseExternalTable.builder().content(new ByteArrayInputStream(new byte[0]))
.build());
request.format(ClickHouseFormat.Avro);
request.table("table1", "query_id1");
request.query("select :a", UUID.randomUUID().toString());
Expand Down Expand Up @@ -419,7 +448,8 @@ public void testSettings() {
Assert.assertEquals(request.getStatements().get(0), "SET enable_optimize_predicate_expression=1");
request.set("log_queries_min_type", "EXCEPTION_WHILE_PROCESSING");
Assert.assertEquals(request.getStatements().size(), 2);
Assert.assertEquals(request.getStatements().get(1), "SET log_queries_min_type='EXCEPTION_WHILE_PROCESSING'");
Assert.assertEquals(request.getStatements().get(1),
"SET log_queries_min_type='EXCEPTION_WHILE_PROCESSING'");
}

@Test(groups = { "unit" })
Expand Down
11 changes: 7 additions & 4 deletions clickhouse-client/src/test/java/com/clickhouse/client/config/ClickHouseOptionTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,20 +61,23 @@ public String getDescription() {

@Test(groups = { "unit" })
public void testFromString() {
Assert.assertThrows(IllegalArgumentException.class,
() -> ClickHouseOption.fromString(null, String.class));
Assert.assertThrows(IllegalArgumentException.class, () -> ClickHouseOption.fromString(null, null));

Assert.assertEquals(ClickHouseOption.fromString(null, String.class), "");
Assert.assertEquals(ClickHouseOption.fromString("", String.class), "");

Assert.assertEquals(ClickHouseOption.fromString("", Boolean.class), Boolean.FALSE);
Assert.assertEquals(ClickHouseOption.fromString("Yes", Boolean.class), Boolean.FALSE);
Assert.assertEquals(ClickHouseOption.fromString("1", boolean.class), true);
Assert.assertEquals(ClickHouseOption.fromString("1", Boolean.class), Boolean.TRUE);
Assert.assertEquals(ClickHouseOption.fromString("true", Boolean.class), Boolean.TRUE);
Assert.assertEquals(ClickHouseOption.fromString("True", Boolean.class), Boolean.TRUE);

Assert.assertEquals(ClickHouseOption.fromString(null, Integer.class), Integer.valueOf(0));
Assert.assertEquals(ClickHouseOption.fromString("", Integer.class), Integer.valueOf(0));
Assert.assertEquals(ClickHouseOption.fromString(" ", 1), 1);
Assert.assertEquals(ClickHouseOption.fromString("0", Integer.class), Integer.valueOf(0));
Assert.assertThrows(IllegalArgumentException.class,
() -> ClickHouseOption.fromString(null, Integer.class));
Assert.assertEquals(ClickHouseOption.fromString("0", int.class), 0);

Assert.assertEquals(ClickHouseOption.fromString("0.1", Float.class), Float.valueOf(0.1F));
Assert.assertEquals(ClickHouseOption.fromString("NaN", Float.class), Float.valueOf(Float.NaN));
Expand Down
Loading