Skip to content

Commit

Permalink
[Bug](datev2) Fix wrong result when use datev2 as partition key (#15094)
Browse files Browse the repository at this point in the history
  • Loading branch information
Gabriel39 committed Dec 15, 2022
1 parent bccea1c commit 5ef4c42
Show file tree
Hide file tree
Showing 8 changed files with 188 additions and 46 deletions.
11 changes: 2 additions & 9 deletions be/src/runtime/raw_value.h
Expand Up @@ -519,18 +519,11 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri
}

case TYPE_DATEV2: {
auto* date_v2_val = (const vectorized::DateV2Value<doris::vectorized::DateV2ValueType>*)v;
char buf[64];
int date_v2_len = date_v2_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, date_v2_len, seed);
return HashUtil::zlib_crc_hash(v, 4, seed);
}

case TYPE_DATETIMEV2: {
auto* date_v2_val =
(const vectorized::DateV2Value<doris::vectorized::DateTimeV2ValueType>*)v;
char buf[64];
int date_v2_len = date_v2_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, date_v2_len, seed);
return HashUtil::zlib_crc_hash(v, 8, seed);
}

case TYPE_DECIMALV2: {
Expand Down
Expand Up @@ -486,9 +486,9 @@ public Object getRealValue() {
return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second;
} else if (type.equals(Type.DATEV2)) {
return (year << 9) | (month << 5) | day;
} else if (type.equals(Type.DATETIMEV2)) {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
} else if (type.isDatetimeV2()) {
return (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
} else {
Preconditions.checkState(false, "invalid date type: " + type);
return -1L;
Expand All @@ -505,8 +505,8 @@ public ByteBuffer getHashValue(PrimitiveType type) {
buffer.order(ByteOrder.LITTLE_ENDIAN);
buffer.putInt(value);
} else if (type == PrimitiveType.DATETIMEV2) {
long value = (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
long value = (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
buffer = ByteBuffer.allocate(8);
buffer.order(ByteOrder.LITTLE_ENDIAN);
buffer.putLong(value);
Expand Down Expand Up @@ -672,9 +672,40 @@ private long makePackedDatetime() {
return packedDatetime;
}

private void fromPackedDatetimeV2(long packedTime) {
microsecond = (packedTime % (1L << 20));
long ymdhms = (packedTime >> 20);
long ymd = ymdhms >> 17;
day = ymd % (1 << 5);
long ym = ymd >> 5;
month = ym % (1 << 4);
year = ym >> 4;

long hms = ymdhms % (1 << 17);
second = hms % (1 << 6);
minute = (hms >> 6) % (1 << 6);
hour = (hms >> 12);
// set default date literal type to DATETIME
// date literal read from meta will set type by flag bit;
this.type = Type.DATETIMEV2;
}

private void fromPackedDateV2(long packedTime) {
day = packedTime % (1 << 5);
long ym = packedTime >> 5;
month = ym % (1 << 4);
year = ym >> 4;

this.type = Type.DATEV2;
}

private long makePackedDatetimeV2() {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
return (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
}

private long makePackedDateV2() {
return ((year << 9) | (month << 5) | day);
}

@Override
Expand All @@ -693,7 +724,7 @@ public void write(DataOutput out) throws IOException {
out.writeInt(((ScalarType) this.type).getScalarScale());
} else if (this.type.equals(Type.DATEV2)) {
out.writeShort(DateLiteralType.DATEV2.value());
out.writeLong(makePackedDatetimeV2());
out.writeLong(makePackedDateV2());
} else {
throw new IOException("Error date literal type : " + type);
}
Expand Down Expand Up @@ -721,14 +752,17 @@ private void fromPackedDatetime(long packedTime) {
public void readFields(DataInput in) throws IOException {
super.readFields(in);
short dateLiteralType = in.readShort();
fromPackedDatetime(in.readLong());
if (dateLiteralType == DateLiteralType.DATETIME.value()) {
fromPackedDatetime(in.readLong());
this.type = Type.DATETIME;
} else if (dateLiteralType == DateLiteralType.DATE.value()) {
fromPackedDatetime(in.readLong());
this.type = Type.DATE;
} else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) {
fromPackedDatetimeV2(in.readLong());
this.type = ScalarType.createDatetimeV2Type(in.readInt());
} else if (dateLiteralType == DateLiteralType.DATEV2.value()) {
fromPackedDateV2(in.readLong());
this.type = Type.DATEV2;
} else {
throw new IOException("Error date literal type : " + type);
Expand Down
Expand Up @@ -59,7 +59,7 @@ public static PartitionKey createInfinityPartitionKey(List<Column> columns, bool
throws AnalysisException {
PartitionKey partitionKey = new PartitionKey();
for (Column column : columns) {
partitionKey.keys.add(LiteralExpr.createInfinity(Type.fromPrimitiveType(column.getDataType()), isMax));
partitionKey.keys.add(LiteralExpr.createInfinity(column.getType(), isMax));
partitionKey.types.add(column.getDataType());
}
return partitionKey;
Expand All @@ -71,15 +71,13 @@ public static PartitionKey createPartitionKey(List<PartitionValue> keys, List<Co
Preconditions.checkArgument(keys.size() <= columns.size());
int i;
for (i = 0; i < keys.size(); ++i) {
partitionKey.keys.add(keys.get(i).getValue(
Type.fromPrimitiveType(columns.get(i).getDataType())));
partitionKey.keys.add(keys.get(i).getValue(columns.get(i).getType()));
partitionKey.types.add(columns.get(i).getDataType());
}

// fill the vacancy with MIN
for (; i < columns.size(); ++i) {
Type type = Type.fromPrimitiveType(columns.get(i).getDataType());
partitionKey.keys.add(LiteralExpr.createInfinity(type, false));
partitionKey.keys.add(LiteralExpr.createInfinity(columns.get(i).getType(), false));
partitionKey.types.add(columns.get(i).getDataType());
}

Expand Down Expand Up @@ -337,7 +335,9 @@ public void readFields(DataInput in) throws IOException {
throw new IOException("type[" + type.name() + "] not supported: ");
}
}
literal.setType(Type.fromPrimitiveType(type));
if (type != PrimitiveType.DATETIMEV2) {
literal.setType(Type.fromPrimitiveType(type));
}
keys.add(literal);
}
}
Expand Down
Expand Up @@ -714,7 +714,7 @@ private void checkDeleteV2(OlapTable table, List<Partition> partitions,
binaryPredicate.setChild(1, LiteralExpr.create(value,
ScalarType.createDatetimeV2Type(ScalarType.MAX_DATETIMEV2_SCALE)));
}
LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType()));
LiteralExpr.create(value, column.getType());
} catch (AnalysisException e) {
// ErrorReport.reportDdlException(ErrorCode.ERR_INVALID_VALUE, value);
throw new DdlException("Invalid column value[" + value + "] for column " + columnName);
Expand All @@ -730,10 +730,10 @@ private void checkDeleteV2(OlapTable table, List<Partition> partitions,
|| column.getDataType() == PrimitiveType.DATEV2
|| column.getDataType() == PrimitiveType.DATETIMEV2) {
DateLiteral dateLiteral = new DateLiteral(value,
Type.fromPrimitiveType(column.getDataType()));
column.getType());
value = dateLiteral.getStringValue();
inPredicate.setChild(i, LiteralExpr.create(value,
Type.fromPrimitiveType(column.getDataType())));
column.getType()));
} else {
LiteralExpr.create(value,
Type.fromPrimitiveType(column.getDataType()));
Expand Down
16 changes: 8 additions & 8 deletions regression-test/data/delete_p0/test_delete.out
Expand Up @@ -14,29 +14,29 @@
-- !sql --
8

-- !sql --
-- !sql1 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql2 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql3 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111
abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111

-- !sql --
-- !sql4 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql5 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql6 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql7 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111
abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111

-- !sql --
-- !sql8 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

39 changes: 39 additions & 0 deletions regression-test/data/partition_p0/test_datev2_partition.out
@@ -0,0 +1,39 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
2022-12-14
2022-12-15
2022-12-16
2022-12-17
2022-12-18
2022-12-19
2022-12-20

-- !select --
2022-12-15

-- !select --
2022-12-16
2022-12-17
2022-12-18
2022-12-19
2022-12-20

-- !select --
2022-12-14T22:22:22.222
2022-12-15T22:22:22.222
2022-12-16T22:22:22.222
2022-12-17T22:22:22.222
2022-12-18T22:22:22.222
2022-12-19T22:22:22.222
2022-12-20T22:22:22.222

-- !select --
2022-12-15T22:22:22.222

-- !select --
2022-12-16T22:22:22.222
2022-12-17T22:22:22.222
2022-12-18T22:22:22.222
2022-12-19T22:22:22.222
2022-12-20T22:22:22.222

22 changes: 11 additions & 11 deletions regression-test/suites/delete_p0/test_delete.groovy
Expand Up @@ -39,36 +39,36 @@ suite("test_delete") {

sql """ INSERT INTO delete_regression_test VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where k2 = '2022-08-16' """
qt_sql """select * from ${tableName};"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """
qt_sql1 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from delete_regression_test where k3 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """
sql """ delete from ${tableName} where k3 = '2022-08-16 11:11:11' """
qt_sql2 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql3 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11.111' """
qt_sql """select * from ${tableName};"""
qt_sql4 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v1 = '2022-08-16' """
qt_sql """select * from ${tableName};"""
qt_sql5 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v2 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql6 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql7 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11.111' """
qt_sql """select * from ${tableName};"""
qt_sql8 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ DROP TABLE IF EXISTS ${tableName} """
Expand Down
76 changes: 76 additions & 0 deletions regression-test/suites/partition_p0/test_datev2_partition.groovy
@@ -0,0 +1,76 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_datev2_partition") {
def tblName1 = "test_datev2_partition1"
sql "drop table if exists ${tblName1}"
sql """
CREATE TABLE `${tblName1}` (
`TIME_STAMP` datev2 NOT NULL COMMENT '采集日期'
) ENGINE=OLAP
DUPLICATE KEY(`TIME_STAMP`)
COMMENT 'OLAP'
PARTITION BY RANGE(`TIME_STAMP`)
(PARTITION p20221214 VALUES [('2022-12-14'), ('2022-12-15')),
PARTITION p20221215 VALUES [('2022-12-15'), ('2022-12-16')),
PARTITION p20221216 VALUES [('2022-12-16'), ('2022-12-17')),
PARTITION p20221217 VALUES [('2022-12-17'), ('2022-12-18')),
PARTITION p20221218 VALUES [('2022-12-18'), ('2022-12-19')),
PARTITION p20221219 VALUES [('2022-12-19'), ('2022-12-20')),
PARTITION p20221220 VALUES [('2022-12-20'), ('2022-12-21')),
PARTITION p20221221 VALUES [('2022-12-21'), ('2022-12-22')))
DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
sql """ insert into ${tblName1} values ('2022-12-14'), ('2022-12-15'), ('2022-12-16'), ('2022-12-17'), ('2022-12-18'), ('2022-12-19'), ('2022-12-20') """

qt_select """ select * from ${tblName1} order by TIME_STAMP """
qt_select """ select * from ${tblName1} WHERE TIME_STAMP = '2022-12-15' order by TIME_STAMP """
qt_select """ select * from ${tblName1} WHERE TIME_STAMP > '2022-12-15' order by TIME_STAMP """
sql "drop table ${tblName1}"

def tblName2 = "test_datev2_partition2"
sql "drop table if exists ${tblName2}"
sql """
CREATE TABLE `${tblName2}` (
`TIME_STAMP` datetimev2(3) NOT NULL COMMENT '采集日期'
) ENGINE=OLAP
DUPLICATE KEY(`TIME_STAMP`)
COMMENT 'OLAP'
PARTITION BY RANGE(`TIME_STAMP`)
(PARTITION p20221214 VALUES [('2022-12-14 11:11:11.111'), ('2022-12-15 11:11:11.111')),
PARTITION p20221215 VALUES [('2022-12-15 11:11:11.111'), ('2022-12-16 11:11:11.111')),
PARTITION p20221216 VALUES [('2022-12-16 11:11:11.111'), ('2022-12-17 11:11:11.111')),
PARTITION p20221217 VALUES [('2022-12-17 11:11:11.111'), ('2022-12-18 11:11:11.111')),
PARTITION p20221218 VALUES [('2022-12-18 11:11:11.111'), ('2022-12-19 11:11:11.111')),
PARTITION p20221219 VALUES [('2022-12-19 11:11:11.111'), ('2022-12-20 11:11:11.111')),
PARTITION p20221220 VALUES [('2022-12-20 11:11:11.111'), ('2022-12-21 11:11:11.111')),
PARTITION p20221221 VALUES [('2022-12-21 11:11:11.111'), ('2022-12-22 11:11:11.111')))
DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
sql """ insert into ${tblName2} values ('2022-12-14 22:22:22.222'), ('2022-12-15 22:22:22.222'), ('2022-12-16 22:22:22.222'), ('2022-12-17 22:22:22.222'), ('2022-12-18 22:22:22.222'), ('2022-12-19 22:22:22.222'), ('2022-12-20 22:22:22.222') """

qt_select """ select * from ${tblName2} order by TIME_STAMP """
qt_select """ select * from ${tblName2} WHERE TIME_STAMP = '2022-12-15 22:22:22.222' order by TIME_STAMP """
qt_select """ select * from ${tblName2} WHERE TIME_STAMP > '2022-12-15 22:22:22.222' order by TIME_STAMP """
sql "drop table ${tblName2}"
}

0 comments on commit 5ef4c42

Please sign in to comment.