diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java index 43e1b53badc91c..fbe79c9d088ef5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PartitionExprUtil.java @@ -261,6 +261,9 @@ private static PartitionValue getPartitionFromDate(Type partitionColumnType, Dat timeString = String.format(DATETIME_FORMATTER, dateLiteral.getYear(), dateLiteral.getMonth(), dateLiteral.getDay(), dateLiteral.getHour(), dateLiteral.getMinute(), dateLiteral.getSecond()); + if (partitionColumnType.isTimeStampTz()) { + timeString = dateLiteral.getStringValue(partitionColumnType); + } } else { throw new AnalysisException( "not support range partition with column type : " + partitionColumnType.toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index ad8819567cb5c5..2a49ae8c137333 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -31,16 +31,13 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.StringLiteral; import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.persist.gson.GsonUtils; -import org.apache.doris.qe.ConnectContext; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; @@ -147,14 +144,7 @@ private static Literal getDateTimeLiteral(String value, Type type) throws Analys } else if (type.isDatetimeV2()) { return new DateTimeV2Literal(value); } else if (type.isTimeStampTz()) { - DateTimeV2Literal literal = new DateTimeV2Literal(value); - DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( - literal, - new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), - new StringLiteral("UTC"))); - return new TimestampTzLiteral((TimeStampTzType) DataType.fromCatalogType(type), - dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(), - dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(), dtV2Lit.getMicroSecond()); + return new TimestampTzLiteral((TimeStampTzType) DataType.fromCatalogType(type), value); } } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java index d6efcb57e3cb32..6c5b81c229c566 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteral.java @@ -63,6 +63,10 @@ public DateTimeLiteral(String s) { this(DateTimeType.INSTANCE, s); } + protected DateTimeLiteral(DateLikeType dataType) { + super(dataType); + } + protected DateTimeLiteral(DateLikeType dataType, String s) { super(dataType); init(s); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java index 621b8d6fa2b99b..dec162909f965d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/StringLikeLiteral.java @@ -21,14 +21,11 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.CastException; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; -import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeType; import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.TimeStampTzType; import org.apache.doris.nereids.types.TimeV2Type; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; import com.google.common.base.Preconditions; @@ -148,20 +145,12 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti return new DateTimeLiteral((DateTimeType) targetType, datetime.year, datetime.month, datetime.day, datetime.hour, datetime.minute, datetime.second, datetime.microSecond); } else if (targetType.isTimeStampTzType()) { - // Explicit offsets must not round-trip through session local time; that loses the selected - // branch in DST fold hours. Wildcard targets still need a concrete scale before parsing. + // Wildcard targets still need a concrete scale before parsing. TimeStampTzType timeStampTzType = (TimeStampTzType) targetType; if (timeStampTzType.getScale() < 0) { timeStampTzType = TimeStampTzType.forTypeFromString(value); } - if (DateTimeChecker.hasTimeZone(value)) { - return new TimestampTzLiteral(timeStampTzType, value); - } - DateTimeV2Literal expression = castToDateTime(DateTimeV2Type.MAX, strictCast, true); - expression = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz(expression, - new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), new StringLiteral("UTC"))); - return new TimestampTzLiteral(timeStampTzType, expression.year, expression.month, - expression.day, expression.hour, expression.minute, expression.second, expression.microSecond); + return new TimestampTzLiteral(timeStampTzType, value); } else if (targetType.isDateTimeV2Type()) { return castToDateTime(targetType, strictCast, true); } else if (targetType.isFloatType()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java index f6a8c25a7daa2d..94567dace1cf88 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java @@ -23,6 +23,7 @@ import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.format.DateTimeChecker; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DateTimeV2Type; @@ -45,7 +46,8 @@ public TimestampTzLiteral(String s) { } public TimestampTzLiteral(TimeStampTzType dateType, String s) { - super(dateType, s); + super(dateType); + init(dateType, s); roundMicroSecond(dateType.getScale()); } @@ -64,6 +66,31 @@ public TimestampTzLiteral(TimeStampTzType dateType, roundMicroSecond(dateType.getScale()); } + private void init(TimeStampTzType dateType, String s) { + if (DateTimeChecker.hasTimeZone(s)) { + super.init(s); + return; + } + + DateTimeV2Literal literal = new DateTimeV2Literal(s); + DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) DateTimeExtractAndTransform.convertTz( + literal, + new StringLiteral(getSessionTimeZone()), + new StringLiteral("UTC")); + this.year = dtV2Lit.year; + this.month = dtV2Lit.month; + this.day = dtV2Lit.day; + this.hour = dtV2Lit.hour; + this.minute = dtV2Lit.minute; + this.second = dtV2Lit.second; + this.microSecond = dtV2Lit.microSecond; + } + + private static String getSessionTimeZone() { + ConnectContext context = ConnectContext.get(); + return context == null ? "UTC" : context.getSessionVariable().timeZone; + } + @Override public TimeStampTzType getDataType() throws UnboundException { return (TimeStampTzType) super.getDataType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java index 51ee92a4e21e11..bd501aa3fd8d98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/PartitionDefinition.java @@ -194,7 +194,7 @@ public void setPartitionTypes(List partitionTypes) { */ protected PartitionValue toLegacyPartitionValueStmt(Expression e) { if (e.isLiteral()) { - return new PartitionValue(((Literal) e).getStringValue(), e.isNullLiteral()); + return new PartitionValue(((Literal) e).toLegacyLiteral().getStringValue(), e.isNullLiteral()); } else if (e instanceof MaxValue) { return PartitionValue.MAX_VALUE; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java index 84f72f954c06bc..fbd2ed1303000a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeCoercionUtils.java @@ -43,7 +43,6 @@ import org.apache.doris.nereids.trees.expressions.SubqueryExpr; import org.apache.doris.nereids.trees.expressions.Subtract; import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; import org.apache.doris.nereids.trees.expressions.functions.scalar.CreateMap; import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral; import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; @@ -110,7 +109,6 @@ import org.apache.doris.nereids.types.coercion.IntegralType; import org.apache.doris.nereids.types.coercion.NumericType; import org.apache.doris.nereids.types.coercion.PrimitiveType; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.qe.SessionVariable; @@ -655,27 +653,13 @@ public static Optional characterLiteralTypeCoercion(String value, Da && DateTimeChecker.isValidDateTime(value)) { ret = DateTimeLiteral.parseDateTimeLiteral(value, true).orElse(null); } else if (dataType.isTimeStampTzType() && DateTimeChecker.isValidDateTime(value)) { - if (DateTimeChecker.hasTimeZone(value)) { - // Signature search can pass TIMESTAMPTZ(*) here. TimestampTzLiteral rounds by scale, - // so derive a concrete scale from the literal before preserving its explicit offset. - TimeStampTzType timeStampTzType = (TimeStampTzType) dataType; - if (timeStampTzType.getScale() < 0) { - timeStampTzType = TimeStampTzType.forTypeFromString(value); - } - ret = new TimestampTzLiteral(timeStampTzType, value); - } else { - DateTimeV2Literal dtV2Lit = (DateTimeV2Literal) DateTimeLiteral - .parseDateTimeLiteral(value, true).orElse(null); - if (dtV2Lit != null) { - dtV2Lit = (DateTimeV2Literal) (DateTimeExtractAndTransform.convertTz( - dtV2Lit, - new StringLiteral(ConnectContext.get().getSessionVariable().timeZone), - new StringLiteral("UTC"))); - ret = new TimestampTzLiteral(dtV2Lit.getYear(), dtV2Lit.getMonth(), dtV2Lit.getDay(), - dtV2Lit.getHour(), dtV2Lit.getMinute(), dtV2Lit.getSecond(), - dtV2Lit.getMicroSecond()); - } + // Signature search can pass TIMESTAMPTZ(*) here. TimestampTzLiteral rounds by scale, + // so derive a concrete scale from the literal before parsing. + TimeStampTzType timeStampTzType = (TimeStampTzType) dataType; + if (timeStampTzType.getScale() < 0) { + timeStampTzType = TimeStampTzType.forTypeFromString(value); } + ret = new TimestampTzLiteral(timeStampTzType, value); } else if ((dataType.isDateV2Type() || dataType.isDateType()) && DateTimeChecker.isValidDateTime(value)) { Result parseResult = DateV2Literal.parseDateLiteral(value, true); if (parseResult.isOk()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java index 6bfe689f4211d0..2166d69a8c3d6a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/PartitionExprUtilTest.java @@ -224,4 +224,48 @@ public void testAutoBucketReusePrevPartitionBuckets() throws Exception { Assertions.assertNotNull(p2); Assertions.assertEquals(p1.getDistributionInfo().getBucketNum(), p2.getDistributionInfo().getBucketNum()); } + + @Test + public void testAutoPartitionDateTruncTimestampTzKeepsUtcBoundary() throws Exception { + String originalTimeZone = connectContext.getSessionVariable().getTimeZone(); + try { + connectContext.getSessionVariable().setTimeZone("+00:00"); + String createOlapTblStmt = "CREATE TABLE test.auto_timestamptz_day_partition (\n" + + " id INT,\n" + + " ts_tz TIMESTAMPTZ(6) NOT NULL\n" + + ")\n" + + "DUPLICATE KEY(id)\n" + + "AUTO PARTITION BY RANGE (date_trunc(ts_tz, 'day')) ()\n" + + "DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + "PROPERTIES(\"replication_num\"=\"1\");"; + + createTable(createOlapTblStmt); + Database db = Env.getCurrentInternalCatalog().getDbOrAnalysisException("test"); + OlapTable table = (OlapTable) db.getTableOrAnalysisException("auto_timestamptz_day_partition"); + + List> partitionValues = new ArrayList<>(); + List values = new ArrayList<>(); + TNullableStringLiteral truncatedValue = new TNullableStringLiteral(); + truncatedValue.setValue("2024-06-15 00:00:00"); + values.add(truncatedValue); + partitionValues.add(values); + + FrontendServiceImpl impl = new FrontendServiceImpl(exeEnv); + TCreatePartitionRequest request = new TCreatePartitionRequest(); + request.setDbId(db.getId()); + request.setTableId(table.getId()); + request.setPartitionValues(partitionValues); + + TCreatePartitionResult result = impl.createPartition(request); + Assertions.assertEquals(TStatusCode.OK, result.getStatus().getStatusCode()); + Assertions.assertNotNull(table.getPartition("p20240615000000")); + + List createTableStmt = new ArrayList<>(); + Env.getDdlStmt(table, createTableStmt, null, null, false, true, -1L); + Assertions.assertTrue(createTableStmt.get(0).contains( + "PARTITION p20240615000000 VALUES [('2024-06-15 00:00:00.000000+00:00'), ('2024-06-16 00:00:00.000000+00:00'))")); + } finally { + connectContext.getSessionVariable().setTimeZone(originalTimeZone); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java index 732c81e5b6b63c..4368ff70c441fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableLikeTest.java @@ -95,6 +95,12 @@ private static void createTableLike(String sql) throws Exception { command.run(connectContext, new StmtExecutor(connectContext, sql)); } + private static String getCreateTableStmt(Table table) { + List createTableStmt = Lists.newArrayList(); + Env.getDdlStmt(table, createTableStmt, null, null, false, true /* hide password */, -1L); + return createTableStmt.get(0); + } + private static void checkTableEqual(Table newTable, Table existedTable, int rollupSize) { List newCreateTableStmt = Lists.newArrayList(); List newAddRollupStmt = Lists.newArrayList(); @@ -384,6 +390,157 @@ public void testAbnormal() { existedDbName3, newTblName3, existedTblName3, 1)); } + @Test + public void testTimestampTzLessThanPartitionCreateTableAcceptsExplicitOffset() throws Exception { + String tableName = "test_timestamptz_less_than_create"; + String createTableSql = "CREATE TABLE test." + tableName + " (\n" + + " `ts` TIMESTAMPTZ(6) NOT NULL,\n" + + " `seq` INT NOT NULL\n" + + ")\n" + + "UNIQUE KEY(`ts`)\n" + + "PARTITION BY RANGE(`ts`) (\n" + + " PARTITION p0 VALUES LESS THAN ('2024-01-15 13:00:00 +00:00'),\n" + + " PARTITION p1 VALUES LESS THAN ('2024-01-15 14:00:00 +00:00')\n" + + ")\n" + + "DISTRIBUTED BY HASH(`ts`) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + ");"; + + String originalTimeZone = connectContext.getSessionVariable().getTimeZone(); + try { + connectContext.getSessionVariable().setTimeZone("+00:00"); + createTable(createTableSql); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + OlapTable table = (OlapTable) db.getTableOrDdlException(tableName); + + String createStmt = getCreateTableStmt(table); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p0 VALUES [('0000-01-01 00:00:00.000000+00:00'), ('2024-01-15 13:00:00.000000+00:00'))")); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p1 VALUES [('2024-01-15 13:00:00.000000+00:00'), ('2024-01-15 14:00:00.000000+00:00'))")); + } finally { + connectContext.getSessionVariable().setTimeZone(originalTimeZone); + } + } + + @Test + public void testTimestampTzRangePartitionCreateTableAcceptsNamedAndLowercaseZones() throws Exception { + String tableName = "test_timestamptz_named_lowercase_zone_create"; + String createTableSql = "CREATE TABLE test." + tableName + " (\n" + + " `ts` TIMESTAMPTZ(6) NOT NULL,\n" + + " `seq` INT NOT NULL\n" + + ")\n" + + "UNIQUE KEY(`ts`)\n" + + "PARTITION BY RANGE(`ts`) (\n" + + " PARTITION p1 VALUES [('2024-01-15 20:00:00Asia/Shanghai'), ('2024-01-15 13:00:00 uTc')),\n" + + " PARTITION p2 VALUES [('2024-01-15 13:00:00 uTc'), ('2024-01-15 22:00:00 Asia/Shanghai'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`ts`) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + ");"; + + String originalTimeZone = connectContext.getSessionVariable().getTimeZone(); + try { + connectContext.getSessionVariable().setTimeZone("America/New_York"); + createTable(createTableSql); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + OlapTable table = (OlapTable) db.getTableOrDdlException(tableName); + + String createStmt = getCreateTableStmt(table); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p1 VALUES [('2024-01-15 12:00:00.000000+00:00'), ('2024-01-15 13:00:00.000000+00:00'))")); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p2 VALUES [('2024-01-15 13:00:00.000000+00:00'), ('2024-01-15 14:00:00.000000+00:00'))")); + } finally { + connectContext.getSessionVariable().setTimeZone(originalTimeZone); + } + } + + @Test + public void testTimestampTzLessThanPartitionCreateTableUsesSessionTimezoneWithoutOffset() throws Exception { + String tableName = "test_timestamptz_less_than_session_tz"; + String createTableSql = "CREATE TABLE test." + tableName + " (\n" + + " `ts` TIMESTAMPTZ(6) NOT NULL,\n" + + " `seq` INT NOT NULL\n" + + ")\n" + + "UNIQUE KEY(`ts`)\n" + + "PARTITION BY RANGE(`ts`) (\n" + + " PARTITION p0 VALUES LESS THAN ('2024-01-15 13:00:00'),\n" + + " PARTITION p1 VALUES LESS THAN ('2024-01-15 14:00:00')\n" + + ")\n" + + "DISTRIBUTED BY HASH(`ts`) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + ");"; + + String originalTimeZone = connectContext.getSessionVariable().getTimeZone(); + try { + connectContext.getSessionVariable().setTimeZone("America/New_York"); + createTable(createTableSql); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + OlapTable table = (OlapTable) db.getTableOrDdlException(tableName); + + String createStmt = getCreateTableStmt(table); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p0 VALUES [('0000-01-01 00:00:00.000000+00:00'), ('2024-01-15 18:00:00.000000+00:00'))")); + Assert.assertTrue(createStmt, createStmt.contains( + "PARTITION p1 VALUES [('2024-01-15 18:00:00.000000+00:00'), ('2024-01-15 19:00:00.000000+00:00'))")); + } finally { + connectContext.getSessionVariable().setTimeZone(originalTimeZone); + } + } + + @Test + public void testTimestampTzRangePartitionCreateLikeKeepsUtcBoundary() throws Exception { + String sourceTableName = "test_timestamptz_range_like_src"; + String cloneTableName = "test_timestamptz_range_like_clone"; + String createSourceTableSql = "CREATE TABLE test." + sourceTableName + " (\n" + + " `ts` TIMESTAMPTZ(6) NOT NULL,\n" + + " `seq` INT NOT NULL\n" + + ")\n" + + "UNIQUE KEY(`ts`)\n" + + "PARTITION BY RANGE(`ts`) (\n" + + " PARTITION p1 VALUES [('2024-01-15 12:00:00 +00:00'), ('2024-01-15 13:00:00 +00:00')),\n" + + " PARTITION p2 VALUES [('2024-01-15 13:00:00 +00:00'), ('2024-01-15 14:00:00 +00:00'))\n" + + ")\n" + + "DISTRIBUTED BY HASH(`ts`) BUCKETS 1\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + ");"; + String createLikeSql = "CREATE TABLE test." + cloneTableName + " LIKE test." + sourceTableName; + + String originalTimeZone = connectContext.getSessionVariable().getTimeZone(); + try { + connectContext.getSessionVariable().setTimeZone("+00:00"); + createTable(createSourceTableSql); + + connectContext.getSessionVariable().setTimeZone("Asia/Shanghai"); + createTableLike(createLikeSql); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + OlapTable sourceTable = (OlapTable) db.getTableOrDdlException(sourceTableName); + OlapTable cloneTable = (OlapTable) db.getTableOrDdlException(cloneTableName); + + String sourceCreateStmt = getCreateTableStmt(sourceTable); + Assert.assertTrue(sourceCreateStmt.contains( + "PARTITION p1 VALUES [('2024-01-15 12:00:00.000000+00:00'), ('2024-01-15 13:00:00.000000+00:00'))")); + Assert.assertTrue(sourceCreateStmt.contains( + "PARTITION p2 VALUES [('2024-01-15 13:00:00.000000+00:00'), ('2024-01-15 14:00:00.000000+00:00'))")); + checkTableEqual(cloneTable, sourceTable, 0); + } finally { + connectContext.getSessionVariable().setTimeZone(originalTimeZone); + } + } + @Test public void checkSyncedTableWithRollup() throws Exception { String createTableWithRollup = "CREATE TABLE IF NOT EXISTS test.table_with_rollup_synced\n" + "(\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java index 1c936da18bcb1f..cae9212facf105 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/PartitionKeyTest.java @@ -17,9 +17,11 @@ package org.apache.doris.catalog; +import org.apache.doris.analysis.DateLiteral; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.qe.ConnectContext; import org.junit.Assert; import org.junit.BeforeClass; @@ -45,6 +47,7 @@ public class PartitionKeyTest { private static Column largeInt; private static Column date; private static Column datetime; + private static Column timestampTz; private static Column charString; private static Column varchar; private static Column bool; @@ -63,6 +66,7 @@ public static void setUp() { largeInt = new Column("largeint", PrimitiveType.LARGEINT); date = new Column("date", PrimitiveType.DATE); datetime = new Column("datetime", PrimitiveType.DATETIME); + timestampTz = new Column("timestamptz", ScalarType.createTimeStampTzType(6), true, null, "", ""); charString = new Column("char", PrimitiveType.CHAR); varchar = new Column("varchar", PrimitiveType.VARCHAR); bool = new Column("bool", PrimitiveType.BOOLEAN); @@ -287,4 +291,98 @@ public void testMaxValueToSql() throws Exception { PartitionKey key = PartitionKey.createInfinityPartitionKey(allColumns, true); Assert.assertEquals("(MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE, MAXVALUE)", key.toSql()); } + + @Test + public void testTimestampTzPartitionKeyKeepsExplicitOffset() throws Exception { + boolean originalRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + ConnectContext context = new ConnectContext(); + context.setThreadLocalInfo(); + try { + context.getSessionVariable().setTimeZone("America/New_York"); + + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("2024-01-15 12:00:00 +00:00")), + Arrays.asList(timestampTz)); + + DateLiteral literal = (DateLiteral) key.getKeys().get(0); + Assert.assertEquals(2024, literal.getYear()); + Assert.assertEquals(1, literal.getMonth()); + Assert.assertEquals(15, literal.getDay()); + Assert.assertEquals(12, literal.getHour()); + Assert.assertEquals(0, literal.getMinute()); + Assert.assertEquals(0, literal.getSecond()); + Assert.assertEquals(0, literal.getMicrosecond()); + Assert.assertTrue(literal.getStringValue().startsWith("2024-01-15 12:00:00")); + Assert.assertTrue(literal.getStringValue().endsWith("+00:00")); + } finally { + ConnectContext.remove(); + FeConstants.runningUnitTest = originalRunningUnitTest; + } + } + + @Test + public void testTimestampTzPartitionKeyAcceptsNamedTimezone() throws Exception { + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("2024-01-15 20:00:00Asia/Shanghai")), + Arrays.asList(timestampTz)); + + DateLiteral literal = (DateLiteral) key.getKeys().get(0); + Assert.assertEquals(2024, literal.getYear()); + Assert.assertEquals(1, literal.getMonth()); + Assert.assertEquals(15, literal.getDay()); + Assert.assertEquals(12, literal.getHour()); + Assert.assertEquals(0, literal.getMinute()); + Assert.assertEquals(0, literal.getSecond()); + Assert.assertEquals(0, literal.getMicrosecond()); + Assert.assertTrue(literal.getStringValue().startsWith("2024-01-15 12:00:00")); + Assert.assertTrue(literal.getStringValue().endsWith("+00:00")); + } + + @Test + public void testTimestampTzPartitionKeyAcceptsLowercaseTimezone() throws Exception { + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("2024-01-15 12:00:00 uTc")), + Arrays.asList(timestampTz)); + + DateLiteral literal = (DateLiteral) key.getKeys().get(0); + Assert.assertEquals(2024, literal.getYear()); + Assert.assertEquals(1, literal.getMonth()); + Assert.assertEquals(15, literal.getDay()); + Assert.assertEquals(12, literal.getHour()); + Assert.assertEquals(0, literal.getMinute()); + Assert.assertEquals(0, literal.getSecond()); + Assert.assertEquals(0, literal.getMicrosecond()); + Assert.assertTrue(literal.getStringValue().startsWith("2024-01-15 12:00:00")); + Assert.assertTrue(literal.getStringValue().endsWith("+00:00")); + } + + @Test + public void testTimestampTzPartitionKeyUsesSessionTimezoneWithoutExplicitOffset() throws Exception { + boolean originalRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + ConnectContext context = new ConnectContext(); + context.setThreadLocalInfo(); + try { + context.getSessionVariable().setTimeZone("America/New_York"); + + PartitionKey key = PartitionKey.createPartitionKey( + Arrays.asList(new PartitionValue("2024-01-15 12:00:00")), + Arrays.asList(timestampTz)); + + DateLiteral literal = (DateLiteral) key.getKeys().get(0); + Assert.assertEquals(2024, literal.getYear()); + Assert.assertEquals(1, literal.getMonth()); + Assert.assertEquals(15, literal.getDay()); + Assert.assertEquals(17, literal.getHour()); + Assert.assertEquals(0, literal.getMinute()); + Assert.assertEquals(0, literal.getSecond()); + Assert.assertEquals(0, literal.getMicrosecond()); + Assert.assertTrue(literal.getStringValue().startsWith("2024-01-15 17:00:00")); + Assert.assertTrue(literal.getStringValue().endsWith("+00:00")); + } finally { + ConnectContext.remove(); + FeConstants.runningUnitTest = originalRunningUnitTest; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java index a1961a61fbb1fe..c50942a38bddb5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java @@ -19,6 +19,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.types.TimeStampTzType; +import org.apache.doris.qe.ConnectContext; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -77,6 +78,27 @@ void testConstructorsAndParsing() { Assertions.assertEquals(0, literal.microSecond); } + @Test + void testTypedConstructorUsesSessionTimezoneWithoutExplicitOffset() { + ConnectContext context = new ConnectContext(); + context.getSessionVariable().setTimeZone("America/New_York"); + context.setThreadLocalInfo(); + try { + TimestampTzLiteral literal = new TimestampTzLiteral(TimeStampTzType.of(6), "2024-01-15 12:00:00"); + + Assertions.assertEquals(2024, literal.year); + Assertions.assertEquals(1, literal.month); + Assertions.assertEquals(15, literal.day); + Assertions.assertEquals(17, literal.hour); + Assertions.assertEquals(0, literal.minute); + Assertions.assertEquals(0, literal.second); + Assertions.assertEquals(0, literal.microSecond); + Assertions.assertEquals(6, literal.getDataType().getScale()); + } finally { + ConnectContext.remove(); + } + } + @Test void testBasicArithmetic() { TimestampTzLiteral base1 = new TimestampTzLiteral("2025-12-31 23:59:59.999999"); diff --git a/regression-test/data/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.out b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.out new file mode 100644 index 00000000000000..18cbd9e8d5399a --- /dev/null +++ b/regression-test/data/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.out @@ -0,0 +1,26 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bug107_create_shift -- +2024-01-15 12:30:00.000000+00:00 1 +2024-01-15 13:30:00.000000+00:00 2 + +-- !bug108_like -- +2024-01-15 12:30:00.000000+00:00 1 +2024-01-15 13:30:00.000000+00:00 2 + +-- !bug110_src -- +2024-01-15 12:30:00.000000+00:00 1 +2024-01-15 13:30:00.000000+00:00 2 + +-- !bug111_src -- +2024-01-15 17:30:00.000000+00:00 1 +2024-01-15 18:30:00.000000+00:00 2 + +-- !bug113_direct_date_trunc_utc -- +2024-06-15 00:00:00.000000+00:00 + +-- !bug113_auto_tz_range -- +2024-06-15 20:00:00.000000+00:00 1 + +-- !bug114_named_lowercase_tz -- +2024-01-15 12:30:00.000000+00:00 1 +2024-01-15 13:30:00.000000+00:00 2 diff --git a/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.groovy b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.groovy new file mode 100644 index 00000000000000..6fe272c2d0cd85 --- /dev/null +++ b/regression-test/suites/datatype_p0/timestamptz/test_timestamptz_partition_boundary_timezone.groovy @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_timestamptz_partition_boundary_timezone") { + def dbName = "timestamptz_partition_boundary_timezone" + def createBoundary = "PARTITION p1 VALUES [('2024-01-15 12:00:00.000000+00:00'), ('2024-01-15 13:00:00.000000+00:00'))" + def createBoundary2 = "PARTITION p2 VALUES [('2024-01-15 13:00:00.000000+00:00'), ('2024-01-15 14:00:00.000000+00:00'))" + + sql "DROP DATABASE IF EXISTS ${dbName}" + sql "CREATE DATABASE ${dbName}" + sql "USE ${dbName}" + + sql "SET enable_nereids_planner = true" + sql "SET enable_fallback_to_original_planner = false" + + sql "SET time_zone = 'America/New_York'" + sql "DROP TABLE IF EXISTS bug107_create_shift" + sql """ + CREATE TABLE bug107_create_shift ( + ts TIMESTAMPTZ(6) NOT NULL, + seq INT NOT NULL + ) + UNIQUE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p1 VALUES [('2024-01-15 12:00:00 +00:00'), ('2024-01-15 13:00:00 +00:00')), + PARTITION p2 VALUES [('2024-01-15 13:00:00 +00:00'), ('2024-01-15 14:00:00 +00:00')) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + def createShift = sql "SHOW CREATE TABLE bug107_create_shift" + assertTrue(createShift[0][1].contains(createBoundary)) + assertTrue(createShift[0][1].contains(createBoundary2)) + + sql "SET time_zone = '+00:00'" + sql """ + INSERT INTO bug107_create_shift VALUES + (CAST('2024-01-15 12:30:00 +00:00' AS TIMESTAMPTZ(6)), 1), + (CAST('2024-01-15 13:30:00 +00:00' AS TIMESTAMPTZ(6)), 2) + """ + + order_qt_bug107_create_shift """ + SELECT CAST(ts AS STRING), seq + FROM bug107_create_shift + ORDER BY seq + """ + + sql "SET time_zone = '+00:00'" + sql "DROP TABLE IF EXISTS bug108_src" + sql """ + CREATE TABLE bug108_src ( + ts TIMESTAMPTZ(6) NOT NULL, + seq INT NOT NULL + ) + UNIQUE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p1 VALUES [('2024-01-15 12:00:00 +00:00'), ('2024-01-15 13:00:00 +00:00')), + PARTITION p2 VALUES [('2024-01-15 13:00:00 +00:00'), ('2024-01-15 14:00:00 +00:00')) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + def srcCreate = sql "SHOW CREATE TABLE bug108_src" + assertTrue(srcCreate[0][1].contains(createBoundary)) + assertTrue(srcCreate[0][1].contains(createBoundary2)) + + sql "SET time_zone = 'Asia/Shanghai'" + sql "DROP TABLE IF EXISTS bug108_like" + sql "CREATE TABLE bug108_like LIKE bug108_src" + + def likeCreate = sql "SHOW CREATE TABLE bug108_like" + assertTrue(likeCreate[0][1].contains(createBoundary)) + assertTrue(likeCreate[0][1].contains(createBoundary2)) + + sql "SET time_zone = '+00:00'" + sql """ + INSERT INTO bug108_like VALUES + (CAST('2024-01-15 12:30:00 +00:00' AS TIMESTAMPTZ(6)), 1), + (CAST('2024-01-15 13:30:00 +00:00' AS TIMESTAMPTZ(6)), 2) + """ + + order_qt_bug108_like """ + SELECT CAST(ts AS STRING), seq + FROM bug108_like + ORDER BY seq + """ + + sql "SET time_zone = '+00:00'" + sql "DROP TABLE IF EXISTS bug110_src" + sql """ + CREATE TABLE bug110_src ( + ts TIMESTAMPTZ(6) NOT NULL, + seq INT NOT NULL + ) + UNIQUE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p0 VALUES LESS THAN ('2024-01-15 13:00:00 +00:00'), + PARTITION p1 VALUES LESS THAN ('2024-01-15 14:00:00 +00:00') + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + def bug110Create = sql "SHOW CREATE TABLE bug110_src" + assertTrue(bug110Create[0][1].contains( + "PARTITION p0 VALUES [('0000-01-01 00:00:00.000000+00:00'), ('2024-01-15 13:00:00.000000+00:00'))")) + assertTrue(bug110Create[0][1].contains( + "PARTITION p1 VALUES [('2024-01-15 13:00:00.000000+00:00'), ('2024-01-15 14:00:00.000000+00:00'))")) + + sql """ + INSERT INTO bug110_src VALUES + (CAST('2024-01-15 12:30:00 +00:00' AS TIMESTAMPTZ(6)), 1), + (CAST('2024-01-15 13:30:00 +00:00' AS TIMESTAMPTZ(6)), 2) + """ + + order_qt_bug110_src """ + SELECT CAST(ts AS STRING), seq + FROM bug110_src + ORDER BY seq + """ + + sql "SET time_zone = 'America/New_York'" + sql "DROP TABLE IF EXISTS bug111_src" + sql """ + CREATE TABLE bug111_src ( + ts TIMESTAMPTZ(6) NOT NULL, + seq INT NOT NULL + ) + UNIQUE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p0 VALUES LESS THAN ('2024-01-15 13:00:00'), + PARTITION p1 VALUES LESS THAN ('2024-01-15 14:00:00') + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + def bug111Create = sql "SHOW CREATE TABLE bug111_src" + assertTrue(bug111Create[0][1].contains( + "PARTITION p0 VALUES [('0000-01-01 00:00:00.000000+00:00'), ('2024-01-15 18:00:00.000000+00:00'))")) + assertTrue(bug111Create[0][1].contains( + "PARTITION p1 VALUES [('2024-01-15 18:00:00.000000+00:00'), ('2024-01-15 19:00:00.000000+00:00'))")) + + sql "SET time_zone = '+00:00'" + sql """ + INSERT INTO bug111_src VALUES + (CAST('2024-01-15 17:30:00 +00:00' AS TIMESTAMPTZ(6)), 1), + (CAST('2024-01-15 18:30:00 +00:00' AS TIMESTAMPTZ(6)), 2) + """ + + order_qt_bug111_src """ + SELECT CAST(ts AS STRING), seq + FROM bug111_src + ORDER BY seq + """ + + sql "SET time_zone = '+00:00'" + sql "DROP TABLE IF EXISTS bug113_auto_tz_range" + sql """ + CREATE TABLE bug113_auto_tz_range ( + id INT, + ts_tz TIMESTAMPTZ(6) NOT NULL + ) + DUPLICATE KEY(id) + AUTO PARTITION BY RANGE (date_trunc(`ts_tz`, 'day')) () + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + order_qt_bug113_direct_date_trunc_utc """ + SELECT CAST(date_trunc(ts, 'day') AS VARCHAR(64)) + FROM ( + SELECT CAST('2024-06-15 20:00:00 +00:00' AS TIMESTAMPTZ(6)) AS ts + ) t + """ + + sql """ + INSERT INTO bug113_auto_tz_range VALUES + (1, CAST('2024-06-15 20:00:00 +00:00' AS TIMESTAMPTZ(6))) + """ + + def bug113Create = sql "SHOW CREATE TABLE bug113_auto_tz_range" + assertTrue(bug113Create[0][1].contains( + "PARTITION p20240615000000 VALUES [('2024-06-15 00:00:00.000000+00:00'), ('2024-06-16 00:00:00.000000+00:00'))")) + + order_qt_bug113_auto_tz_range """ + SELECT CAST(ts_tz AS STRING), id + FROM bug113_auto_tz_range + ORDER BY id + """ + + sql "SET time_zone = 'America/New_York'" + sql "DROP TABLE IF EXISTS bug114_named_lowercase_tz" + sql """ + CREATE TABLE bug114_named_lowercase_tz ( + ts TIMESTAMPTZ(6) NOT NULL, + seq INT NOT NULL + ) + UNIQUE KEY(ts) + PARTITION BY RANGE(ts) ( + PARTITION p1 VALUES [('2024-01-15 20:00:00Asia/Shanghai'), ('2024-01-15 13:00:00 uTc')), + PARTITION p2 VALUES [('2024-01-15 13:00:00 uTc'), ('2024-01-15 22:00:00 Asia/Shanghai')) + ) + DISTRIBUTED BY HASH(ts) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + def bug114Create = sql "SHOW CREATE TABLE bug114_named_lowercase_tz" + assertTrue(bug114Create[0][1].contains(createBoundary)) + assertTrue(bug114Create[0][1].contains(createBoundary2)) + + sql "SET time_zone = '+00:00'" + sql """ + INSERT INTO bug114_named_lowercase_tz VALUES + (CAST('2024-01-15 12:30:00 +00:00' AS TIMESTAMPTZ(6)), 1), + (CAST('2024-01-15 13:30:00 +00:00' AS TIMESTAMPTZ(6)), 2) + """ + + order_qt_bug114_named_lowercase_tz """ + SELECT CAST(ts AS STRING), seq + FROM bug114_named_lowercase_tz + ORDER BY seq + """ +} \ No newline at end of file