diff --git a/be/src/runtime/raw_value.h b/be/src/runtime/raw_value.h index 70d12c60d53add..ceb722e84719fe 100644 --- a/be/src/runtime/raw_value.h +++ b/be/src/runtime/raw_value.h @@ -519,18 +519,11 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri } case TYPE_DATEV2: { - auto* date_v2_val = (const vectorized::DateV2Value*)v; - char buf[64]; - int date_v2_len = date_v2_val->to_buffer(buf); - return HashUtil::zlib_crc_hash(buf, date_v2_len, seed); + return HashUtil::zlib_crc_hash(v, 4, seed); } case TYPE_DATETIMEV2: { - auto* date_v2_val = - (const vectorized::DateV2Value*)v; - char buf[64]; - int date_v2_len = date_v2_val->to_buffer(buf); - return HashUtil::zlib_crc_hash(buf, date_v2_len, seed); + return HashUtil::zlib_crc_hash(v, 8, seed); } case TYPE_DECIMALV2: { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index b73cc7cd24920e..2a6efebd4072bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -486,9 +486,9 @@ public Object getRealValue() { return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; } else if (type.equals(Type.DATEV2)) { return (year << 9) | (month << 5) | day; - } else if (type.equals(Type.DATETIMEV2)) { - return (year << 50) | (month << 46) | (day << 41) | (hour << 36) - | (minute << 30) | (second << 24) | microsecond; + } else if (type.isDatetimeV2()) { + return (year << 46) | (month << 42) | (day << 37) | (hour << 32) + | (minute << 26) | (second << 20) | (microsecond % (1 << 20)); } else { Preconditions.checkState(false, "invalid date type: " + type); return -1L; @@ -505,8 +505,8 @@ public ByteBuffer getHashValue(PrimitiveType type) { buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putInt(value); } else if (type == PrimitiveType.DATETIMEV2) { - long value = (year << 50) | (month << 46) | (day << 41) | (hour << 36) - | (minute << 30) | (second << 24) | microsecond; + long value = (year << 46) | (month << 42) | (day << 37) | (hour << 32) + | (minute << 26) | (second << 20) | (microsecond % (1 << 20)); buffer = ByteBuffer.allocate(8); buffer.order(ByteOrder.LITTLE_ENDIAN); buffer.putLong(value); @@ -672,9 +672,40 @@ private long makePackedDatetime() { return packedDatetime; } + private void fromPackedDatetimeV2(long packedTime) { + microsecond = (packedTime % (1L << 20)); + long ymdhms = (packedTime >> 20); + long ymd = ymdhms >> 17; + day = ymd % (1 << 5); + long ym = ymd >> 5; + month = ym % (1 << 4); + year = ym >> 4; + + long hms = ymdhms % (1 << 17); + second = hms % (1 << 6); + minute = (hms >> 6) % (1 << 6); + hour = (hms >> 12); + // set default date literal type to DATETIME + // date literal read from meta will set type by flag bit; + this.type = Type.DATETIMEV2; + } + + private void fromPackedDateV2(long packedTime) { + day = packedTime % (1 << 5); + long ym = packedTime >> 5; + month = ym % (1 << 4); + year = ym >> 4; + + this.type = Type.DATEV2; + } + private long makePackedDatetimeV2() { - return (year << 50) | (month << 46) | (day << 41) | (hour << 36) - | (minute << 30) | (second << 24) | microsecond; + return (year << 46) | (month << 42) | (day << 37) | (hour << 32) + | (minute << 26) | (second << 20) | (microsecond % (1 << 20)); + } + + private long makePackedDateV2() { + return ((year << 9) | (month << 5) | day); } @Override @@ -693,7 +724,7 @@ public void write(DataOutput out) throws IOException { out.writeInt(((ScalarType) this.type).getScalarScale()); } else if (this.type.equals(Type.DATEV2)) { out.writeShort(DateLiteralType.DATEV2.value()); - out.writeLong(makePackedDatetimeV2()); + out.writeLong(makePackedDateV2()); } else { throw new IOException("Error date literal type : " + type); } @@ -721,14 +752,17 @@ private void fromPackedDatetime(long packedTime) { public void readFields(DataInput in) throws IOException { super.readFields(in); short dateLiteralType = in.readShort(); - fromPackedDatetime(in.readLong()); if (dateLiteralType == DateLiteralType.DATETIME.value()) { + fromPackedDatetime(in.readLong()); this.type = Type.DATETIME; } else if (dateLiteralType == DateLiteralType.DATE.value()) { + fromPackedDatetime(in.readLong()); this.type = Type.DATE; } else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) { + fromPackedDatetimeV2(in.readLong()); this.type = ScalarType.createDatetimeV2Type(in.readInt()); } else if (dateLiteralType == DateLiteralType.DATEV2.value()) { + fromPackedDateV2(in.readLong()); this.type = Type.DATEV2; } else { throw new IOException("Error date literal type : " + type); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java index f773209cd19efc..68b49c29740ae3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java @@ -59,7 +59,7 @@ public static PartitionKey createInfinityPartitionKey(List columns, bool throws AnalysisException { PartitionKey partitionKey = new PartitionKey(); for (Column column : columns) { - partitionKey.keys.add(LiteralExpr.createInfinity(Type.fromPrimitiveType(column.getDataType()), isMax)); + partitionKey.keys.add(LiteralExpr.createInfinity(column.getType(), isMax)); partitionKey.types.add(column.getDataType()); } return partitionKey; @@ -71,15 +71,13 @@ public static PartitionKey createPartitionKey(List keys, List partitions, binaryPredicate.setChild(1, LiteralExpr.create(value, ScalarType.createDatetimeV2Type(ScalarType.MAX_DATETIMEV2_SCALE))); } - LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType())); + LiteralExpr.create(value, column.getType()); } catch (AnalysisException e) { // ErrorReport.reportDdlException(ErrorCode.ERR_INVALID_VALUE, value); throw new DdlException("Invalid column value[" + value + "] for column " + columnName); @@ -730,10 +730,10 @@ private void checkDeleteV2(OlapTable table, List partitions, || column.getDataType() == PrimitiveType.DATEV2 || column.getDataType() == PrimitiveType.DATETIMEV2) { DateLiteral dateLiteral = new DateLiteral(value, - Type.fromPrimitiveType(column.getDataType())); + column.getType()); value = dateLiteral.getStringValue(); inPredicate.setChild(i, LiteralExpr.create(value, - Type.fromPrimitiveType(column.getDataType()))); + column.getType())); } else { LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType())); diff --git a/regression-test/data/delete_p0/test_delete.out b/regression-test/data/delete_p0/test_delete.out index 59317fdefc3689..37c01de74e492f 100644 --- a/regression-test/data/delete_p0/test_delete.out +++ b/regression-test/data/delete_p0/test_delete.out @@ -14,29 +14,29 @@ -- !sql -- 8 --- !sql -- +-- !sql1 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 --- !sql -- +-- !sql2 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 --- !sql -- +-- !sql3 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 --- !sql -- +-- !sql4 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 --- !sql -- +-- !sql5 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 --- !sql -- +-- !sql6 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 --- !sql -- +-- !sql7 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 --- !sql -- +-- !sql8 -- abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 diff --git a/regression-test/data/partition_p0/test_datev2_partition.out b/regression-test/data/partition_p0/test_datev2_partition.out new file mode 100644 index 00000000000000..224711371f6dcc --- /dev/null +++ b/regression-test/data/partition_p0/test_datev2_partition.out @@ -0,0 +1,39 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select -- +2022-12-14 +2022-12-15 +2022-12-16 +2022-12-17 +2022-12-18 +2022-12-19 +2022-12-20 + +-- !select -- +2022-12-15 + +-- !select -- +2022-12-16 +2022-12-17 +2022-12-18 +2022-12-19 +2022-12-20 + +-- !select -- +2022-12-14T22:22:22.222 +2022-12-15T22:22:22.222 +2022-12-16T22:22:22.222 +2022-12-17T22:22:22.222 +2022-12-18T22:22:22.222 +2022-12-19T22:22:22.222 +2022-12-20T22:22:22.222 + +-- !select -- +2022-12-15T22:22:22.222 + +-- !select -- +2022-12-16T22:22:22.222 +2022-12-17T22:22:22.222 +2022-12-18T22:22:22.222 +2022-12-19T22:22:22.222 +2022-12-20T22:22:22.222 + diff --git a/regression-test/suites/delete_p0/test_delete.groovy b/regression-test/suites/delete_p0/test_delete.groovy index 10f75c4695102b..90ecb6db028c9d 100644 --- a/regression-test/suites/delete_p0/test_delete.groovy +++ b/regression-test/suites/delete_p0/test_delete.groovy @@ -39,36 +39,36 @@ suite("test_delete") { sql """ INSERT INTO delete_regression_test VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ sql """ delete from ${tableName} where k2 = '2022-08-16' """ - qt_sql """select * from ${tableName};""" - sql """ delete from delete_regression_test where k1 = 'abcdef' """ + qt_sql1 """select * from ${tableName} ORDER BY k2;""" + sql """ delete from ${tableName} where k1 = 'abcdef' """ sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ - sql """ delete from delete_regression_test where k3 = '2022-08-16 11:11:11' """ - qt_sql """select * from ${tableName};""" - sql """ delete from delete_regression_test where k1 = 'abcdef' """ + sql """ delete from ${tableName} where k3 = '2022-08-16 11:11:11' """ + qt_sql2 """select * from ${tableName} ORDER BY k2;""" + sql """ delete from ${tableName} where k1 = 'abcdef' """ sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11' """ - qt_sql """select * from ${tableName};""" + qt_sql3 """select * from ${tableName} ORDER BY k2;""" sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11.111' """ - qt_sql """select * from ${tableName};""" + qt_sql4 """select * from ${tableName} ORDER BY k2;""" sql """ delete from delete_regression_test where k1 = 'abcdef' """ sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ sql """ delete from ${tableName} where v1 = '2022-08-16' """ - qt_sql """select * from ${tableName};""" + qt_sql5 """select * from ${tableName} ORDER BY k2;""" sql """ delete from delete_regression_test where k1 = 'abcdef' """ sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ sql """ delete from ${tableName} where v2 = '2022-08-16 11:11:11' """ - qt_sql """select * from ${tableName};""" + qt_sql6 """select * from ${tableName} ORDER BY k2;""" sql """ delete from delete_regression_test where k1 = 'abcdef' """ sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """ sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11' """ - qt_sql """select * from ${tableName};""" + qt_sql7 """select * from ${tableName} ORDER BY k2;""" sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11.111' """ - qt_sql """select * from ${tableName};""" + qt_sql8 """select * from ${tableName} ORDER BY k2;""" sql """ delete from delete_regression_test where k1 = 'abcdef' """ sql """ DROP TABLE IF EXISTS ${tableName} """ diff --git a/regression-test/suites/partition_p0/test_datev2_partition.groovy b/regression-test/suites/partition_p0/test_datev2_partition.groovy new file mode 100644 index 00000000000000..600b820684c240 --- /dev/null +++ b/regression-test/suites/partition_p0/test_datev2_partition.groovy @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_datev2_partition") { + def tblName1 = "test_datev2_partition1" + sql "drop table if exists ${tblName1}" + sql """ + CREATE TABLE `${tblName1}` ( + `TIME_STAMP` datev2 NOT NULL COMMENT '采集日期' + ) ENGINE=OLAP + DUPLICATE KEY(`TIME_STAMP`) + COMMENT 'OLAP' + PARTITION BY RANGE(`TIME_STAMP`) + (PARTITION p20221214 VALUES [('2022-12-14'), ('2022-12-15')), + PARTITION p20221215 VALUES [('2022-12-15'), ('2022-12-16')), + PARTITION p20221216 VALUES [('2022-12-16'), ('2022-12-17')), + PARTITION p20221217 VALUES [('2022-12-17'), ('2022-12-18')), + PARTITION p20221218 VALUES [('2022-12-18'), ('2022-12-19')), + PARTITION p20221219 VALUES [('2022-12-19'), ('2022-12-20')), + PARTITION p20221220 VALUES [('2022-12-20'), ('2022-12-21')), + PARTITION p20221221 VALUES [('2022-12-21'), ('2022-12-22'))) + DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ insert into ${tblName1} values ('2022-12-14'), ('2022-12-15'), ('2022-12-16'), ('2022-12-17'), ('2022-12-18'), ('2022-12-19'), ('2022-12-20') """ + + qt_select """ select * from ${tblName1} order by TIME_STAMP """ + qt_select """ select * from ${tblName1} WHERE TIME_STAMP = '2022-12-15' order by TIME_STAMP """ + qt_select """ select * from ${tblName1} WHERE TIME_STAMP > '2022-12-15' order by TIME_STAMP """ + sql "drop table ${tblName1}" + + def tblName2 = "test_datev2_partition2" + sql "drop table if exists ${tblName2}" + sql """ + CREATE TABLE `${tblName2}` ( + `TIME_STAMP` datetimev2(3) NOT NULL COMMENT '采集日期' + ) ENGINE=OLAP + DUPLICATE KEY(`TIME_STAMP`) + COMMENT 'OLAP' + PARTITION BY RANGE(`TIME_STAMP`) + (PARTITION p20221214 VALUES [('2022-12-14 11:11:11.111'), ('2022-12-15 11:11:11.111')), + PARTITION p20221215 VALUES [('2022-12-15 11:11:11.111'), ('2022-12-16 11:11:11.111')), + PARTITION p20221216 VALUES [('2022-12-16 11:11:11.111'), ('2022-12-17 11:11:11.111')), + PARTITION p20221217 VALUES [('2022-12-17 11:11:11.111'), ('2022-12-18 11:11:11.111')), + PARTITION p20221218 VALUES [('2022-12-18 11:11:11.111'), ('2022-12-19 11:11:11.111')), + PARTITION p20221219 VALUES [('2022-12-19 11:11:11.111'), ('2022-12-20 11:11:11.111')), + PARTITION p20221220 VALUES [('2022-12-20 11:11:11.111'), ('2022-12-21 11:11:11.111')), + PARTITION p20221221 VALUES [('2022-12-21 11:11:11.111'), ('2022-12-22 11:11:11.111'))) + DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + sql """ insert into ${tblName2} values ('2022-12-14 22:22:22.222'), ('2022-12-15 22:22:22.222'), ('2022-12-16 22:22:22.222'), ('2022-12-17 22:22:22.222'), ('2022-12-18 22:22:22.222'), ('2022-12-19 22:22:22.222'), ('2022-12-20 22:22:22.222') """ + + qt_select """ select * from ${tblName2} order by TIME_STAMP """ + qt_select """ select * from ${tblName2} WHERE TIME_STAMP = '2022-12-15 22:22:22.222' order by TIME_STAMP """ + qt_select """ select * from ${tblName2} WHERE TIME_STAMP > '2022-12-15 22:22:22.222' order by TIME_STAMP """ + sql "drop table ${tblName2}" +}