Skip to content

Commit

Permalink
chore(sql): InTimestampTimestampFunctionFactory Varchar support (#4415)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jerrinot
jerrinot committed Apr 19, 2024
1 parent 9626172 commit c26ee23
Show file tree
Hide file tree
Showing 8 changed files with 168 additions and 52 deletions.
4 changes: 2 additions & 2 deletions core/src/main/java/io/questdb/griffin/FunctionParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -560,7 +560,7 @@ private Function createFunction(
&& argCount == 2
&& args.getQuick(0).isUndefined()
&& args.getQuick(1).isConstant()
) skipAssigningType: {
) skipAssigningType:{
final Function undefinedArg = args.getQuick(0);
final int castToType = args.getQuick(1).getType();
final int assignType;
Expand Down Expand Up @@ -848,7 +848,7 @@ private Function createFunction(
} else if (sigArgTypeTag == ColumnType.INT) {
args.setQuick(k, IntConstant.NULL);
}
} else if ((argTypeTag == ColumnType.STRING || argTypeTag == ColumnType.SYMBOL) && arg.isConstant()) {
} else if ((argTypeTag == ColumnType.STRING || argTypeTag == ColumnType.SYMBOL || argTypeTag == ColumnType.VARCHAR) && arg.isConstant()) {
if (sigArgTypeTag == ColumnType.TIMESTAMP) {
int position = argPositions.getQuick(k);
long timestamp = parseTimestamp(arg.getStrA(null), position);
Expand Down
9 changes: 9 additions & 0 deletions ...src/main/java/io/questdb/griffin/engine/functions/bool/InTimestampStrFunctionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,14 @@ public Function getRight() {
public boolean isReadThreadSafe() {
return false;
}

@Override
public void toPlan(PlanSink sink) {
sink.val(left);
if (negated) {
sink.val(" not");
}
sink.val(" in ").val(right);
}
}
}
5 changes: 4 additions & 1 deletion ...in/java/io/questdb/griffin/engine/functions/bool/InTimestampTimestampFunctionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ public Function newInstance(int position, ObjList<Function> args, IntList argPos
case ColumnType.INT:
case ColumnType.STRING:
case ColumnType.SYMBOL:
case ColumnType.VARCHAR:
break;
default:
throw SqlException.position(0).put("cannot compare TIMESTAMP with type ").put(ColumnType.nameOf(func.getType()));
Expand All @@ -80,7 +81,7 @@ public Function newInstance(int position, ObjList<Function> args, IntList argPos
return new InTimestampConstFunction(args.getQuick(0), parseToTs(args, argPositions));
}

if (args.size() == 2 && ColumnType.isString(args.get(1).getType())) {
if (args.size() == 2 && (ColumnType.isString(args.get(1).getType()) || ColumnType.isVarchar(args.get(1).getType()))) {
// special case - one argument and it a string
return new InTimestampStrFunctionFactory.EqTimestampStrFunction(args.get(0), args.get(1));
}
Expand Down Expand Up @@ -108,6 +109,7 @@ private LongList parseToTs(ObjList<Function> args, IntList argPositions) throws
break;
case ColumnType.STRING:
case ColumnType.SYMBOL:
case ColumnType.VARCHAR:
CharSequence tsValue = func.getStrA(null);
val = (tsValue != null) ? tryParseTimestamp(tsValue, argPositions.getQuick(i)) : Numbers.LONG_NaN;
break;
Expand Down Expand Up @@ -176,6 +178,7 @@ public boolean getBool(Record rec) {
break;
case ColumnType.STRING:
case ColumnType.SYMBOL:
case ColumnType.VARCHAR:
CharSequence str = func.getStrA(rec);
val = str != null ? IntervalUtils.tryParseTimestamp(str) : Numbers.LONG_NaN;
break;
Expand Down
34 changes: 34 additions & 0 deletions ...main/java/io/questdb/griffin/engine/functions/bool/InTimestampVarcharFunctionFactory.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (c) 2014-2019 Appsicle
* Copyright (c) 2019-2024 QuestDB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package io.questdb.griffin.engine.functions.bool;

public final class InTimestampVarcharFunctionFactory extends InTimestampStrFunctionFactory {

@Override
public String getSignature() {
// we have explicit in(NØ) otherwise the function parser would match in(NV) instead
return "in(NØ)";
}
}
1 change: 1 addition & 0 deletions core/src/main/java/module-info.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,7 @@
io.questdb.griffin.engine.functions.bool.InLongFunctionFactory,
io.questdb.griffin.engine.functions.bool.InSymbolFunctionFactory,
io.questdb.griffin.engine.functions.bool.InTimestampStrFunctionFactory,
io.questdb.griffin.engine.functions.bool.InTimestampVarcharFunctionFactory,
io.questdb.griffin.engine.functions.bool.InTimestampTimestampFunctionFactory,
io.questdb.griffin.engine.functions.bool.BetweenTimestampFunctionFactory,
io.questdb.griffin.engine.functions.bool.InUuidFunctionFactory,
Expand Down
1 change: 1 addition & 0 deletions core/src/main/resources/META-INF/services/io.questdb.griffin.FunctionFactory
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,7 @@ io.questdb.griffin.engine.functions.bool.InDoubleFunctionFactory
io.questdb.griffin.engine.functions.bool.InLongFunctionFactory
io.questdb.griffin.engine.functions.bool.InSymbolFunctionFactory
io.questdb.griffin.engine.functions.bool.InTimestampStrFunctionFactory
io.questdb.griffin.engine.functions.bool.InTimestampVarcharFunctionFactory
io.questdb.griffin.engine.functions.bool.InTimestampTimestampFunctionFactory
io.questdb.griffin.engine.functions.bool.BetweenTimestampFunctionFactory
io.questdb.griffin.engine.functions.bool.InUuidFunctionFactory
Expand Down
69 changes: 69 additions & 0 deletions core/src/test/java/io/questdb/test/griffin/TimestampQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1221,6 +1221,75 @@ public void testTimestampStringComparisonInString() throws Exception {
});
}

@Test
public void testTimestampStringComparisonInVarchar() throws Exception {
assertMemoryLeak(() -> {
// create table
String createStmt = "create table tt (dts timestamp, nts timestamp) timestamp(dts)";
ddl(createStmt);

// insert same values to dts (designated) as nts (non-designated) timestamp
insert("insert into tt " +
"select timestamp_sequence(1577836800000000L, 60*60*1000000L), timestamp_sequence(1577836800000000L, 60*60*1000000L) " +
"from long_sequence(48L)");

String expected;
// not in period
expected = "min\tmax\n" +
"2020-01-02T00:00:00.000000Z\t2020-01-02T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts not in '2020-01-01'::varchar");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in '2020-01-01'::varchar");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts not in cast(('2020-01' || '-02') as varchar)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in cast(('2020-01-' || rnd_str('01', '01')) as varchar)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T12:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in ('2020-01-01T12:00'::varchar, '2020-01-01'::varchar)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T00:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where dts in ('2020-01-01'::varchar, '2020-01-03'::varchar) ");

expected = "min\tmax\n" +
"2020-01-02T00:00:00.000000Z\t2020-01-02T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in cast('2020-01' || '-02' as varchar)");

expected = "dts\tnts\n" +
"2020-01-02T00:00:00.000000Z\t2020-01-02T00:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select * from tt where nts in (NULL, cast('2020-01-05' as TIMESTAMP), '2020-01-02'::varchar, NULL)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-02T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in (now(),'2020-01-01'::varchar,1234567,1234567L,CAST('2020-01-01' as TIMESTAMP),NULL::varchar,nts)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-01T00:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts in (now(),'2020-01-01'::varchar)");

expected = "dts\tnts\n";
assertTimestampTtQuery(expected, "select * from tt where CAST(NULL as TIMESTAMP) in ('2020-01-02'::varchar, now())");

expected = "dts\tnts\n";
assertTimestampTtQuery(expected, "select * from tt where CAST(NULL as TIMESTAMP) in ('2020-01-02'::varchar, '2020-01-01'::varchar)");

expected = "dts\tnts\n";
assertTimestampTtQuery(expected, "select * from tt where nts in (now() || 'invalid'::varchar)");

expected = "min\tmax\n" +
"2020-01-01T00:00:00.000000Z\t2020-01-02T23:00:00.000000Z\n";
assertTimestampTtQuery(expected, "select min(nts), max(nts) from tt where nts not in (now() || 'invalid'::varchar)");
});
}

@Test
public void testTimestampStringComparisonInvalidValue() throws Exception {
assertMemoryLeak(() -> {
Expand Down
97 changes: 48 additions & 49 deletions core/src/test/java/io/questdb/test/griffin/WhereClauseParserTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,52 +174,6 @@ public void tearDown() {
super.tearDown(false);
}

@Test
public void testVarcharTimestampParseBasic() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar")));
}

@Test
public void testVarcharTimestampParseOperators() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000001Z, hi=294247-01-10T04:00:54.775807Z}]", intervalToString(modelOf("timestamp > '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=, hi=2024-02-28T23:59:59.999999Z}]", intervalToString(modelOf("timestamp < '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp <= '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=294247-01-10T04:00:54.775807Z}]", intervalToString(modelOf("timestamp >= '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-03-01T00:00:00.000000Z}]", intervalToString(modelOf("timestamp between '2024-02-29'::varchar and '2024-03-01'::varchar")));
}

@Test
public void testVarcharTimestampParseCompoundExpr() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar and timestamp <= '2024-03-01'::varchar")));
}

@Test
public void testVarcharPracticalParsing() throws Exception {
assertMemoryLeak(() -> {
ddl("create table testVarcharPracticalParsing ( a varchar, ts timestamp) timestamp(ts)");
assertPlan(
"select * from testVarcharPracticalParsing where\n" +
"ts = '2024-02-29'::varchar or ts <= '2024-03-01'::varchar",
"Async Filter workers: 1\n" +
" filter: (ts='2024-02-29' or '2024-03-01'>=ts)\n" +
" DataFrame\n" +
" Row forward scan\n" +
" Frame forward scan on: testVarcharPracticalParsing\n"
);

assertPlan(
"select * from testVarcharPracticalParsing where\n" +
"(ts = '2024-02-29'::varchar or ts <= '2024-03-01'::varchar) or ts = '2024-05-01'::varchar",
"Async Filter workers: 1\n" +
" filter: ((ts='2024-02-29' or '2024-03-01'>=ts) or ts='2024-05-01')\n" +
" DataFrame\n" +
" Row forward scan\n" +
" Frame forward scan on: testVarcharPracticalParsing\n"
);
});
}

@Test
public void testAndBranchWithNonIndexedField() throws Exception {
IntrinsicModel m = modelOf("timestamp between '2014-01-01T12:30:00.000Z' and '2014-01-02T12:30:00.000Z' and bid > 100");
Expand Down Expand Up @@ -1479,7 +1433,7 @@ public void testInAndNotInBindVariable8b() throws Exception {
Assert.assertEquals(IntrinsicModel.UNDEFINED, m.intrinsicValue);
Assert.assertEquals("[a]", keyValueFuncsToString(m.keyValueFuncs));
Assert.assertEquals("[]", keyValueFuncsToString(m.keyExcludedValueFuncs));
assertFilter(m, "'$1' sym in");//either concrete values or bind variables have to be double checked
assertFilter(m, "'$1' sym in");//either concrete values or bind variables have to be double checked
}

@Test//bind variable value is unknown so it can't be merged with any other value except the same in not IN set
Expand Down Expand Up @@ -2915,7 +2869,7 @@ public void testTwoIntervalSources() throws Exception {
TestUtils.assertEquals("[]", intervalToString(m));
}

@Test
@Test
public void testTwoIntervalSourcesVarchar() throws Exception {
IntrinsicModel m = modelOf("timestamp in '2014-06-20T13:25:00.000Z;10m;2d;5'::varchar and timestamp IN '2015-06-20T13:25:00.000Z;10m;2d;5'::varchar");
Assert.assertEquals(IntrinsicModel.FALSE, m.intrinsicValue);
Expand Down Expand Up @@ -3071,6 +3025,52 @@ public void testUnindexedPreferredInVsIndexedVarchar() throws SqlException {
TestUtils.assertEquals("[1,2]", keyValueFuncsToString(m.keyValueFuncs));
}

@Test
public void testVarcharPracticalParsing() throws Exception {
assertMemoryLeak(() -> {
ddl("create table testVarcharPracticalParsing ( a string, ts timestamp) timestamp(ts)");
assertPlan(
"select * from testVarcharPracticalParsing where\n" +
"ts = '2024-02-29' or ts <= '2024-03-01'",
"Async Filter workers: 1\n" +
" filter: (ts=1709164800000000 or 1709251200000000>=ts)\n" +
" DataFrame\n" +
" Row forward scan\n" +
" Frame forward scan on: testVarcharPracticalParsing\n"
);

assertPlan(
"select * from testVarcharPracticalParsing where\n" +
"(ts = '2024-02-29'::varchar or ts <= '2024-03-01'::varchar) or ts = '2024-05-01'::varchar",
"Async Filter workers: 1\n" +
" filter: ((ts=1709164800000000 or 1709251200000000>=ts) or ts=1714521600000000)\n" +
" DataFrame\n" +
" Row forward scan\n" +
" Frame forward scan on: testVarcharPracticalParsing\n"
);
});
}

@Test
public void testVarcharTimestampParseBasic() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar")));
}

@Test
public void testVarcharTimestampParseCompoundExpr() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar and timestamp <= '2024-03-01'::varchar")));
}

@Test
public void testVarcharTimestampParseOperators() throws SqlException {
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp = '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000001Z, hi=294247-01-10T04:00:54.775807Z}]", intervalToString(modelOf("timestamp > '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=, hi=2024-02-28T23:59:59.999999Z}]", intervalToString(modelOf("timestamp < '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=, hi=2024-02-29T00:00:00.000000Z}]", intervalToString(modelOf("timestamp <= '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=294247-01-10T04:00:54.775807Z}]", intervalToString(modelOf("timestamp >= '2024-02-29'::varchar")));
TestUtils.assertEquals("[{lo=2024-02-29T00:00:00.000000Z, hi=2024-03-01T00:00:00.000000Z}]", intervalToString(modelOf("timestamp between '2024-02-29'::varchar and '2024-03-01'::varchar")));
}

@Test
public void testWrongTypeConstFunctionDateGreater() {
try {
Expand All @@ -3092,7 +3092,6 @@ public void testWrongTypeConstFunctionDateLess() {
}



private static void swap(String[] arr, int i, int j) {
String tmp = arr[i];
arr[i] = arr[j];
Expand Down

0 comments on commit c26ee23

Please sign in to comment.