Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Bug](datev2) Fix wrong result when use datev2 as partition key #15094

Merged
merged 3 commits into from Dec 15, 2022
Merged

[Bug](datev2) Fix wrong result when use datev2 as partition key #15094

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6026b18
[Bug](datev2) Fix wrong result when use datev2 as partition key
Gabriel39 Dec 15, 2022
80a2848
update
Gabriel39 Dec 15, 2022
a5a1a3e
update
Gabriel39 Dec 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 2 additions & 9 deletions be/src/runtime/raw_value.h
View file
Open in desktop
Expand Up @@ -519,18 +519,11 @@ inline uint32_t RawValue::zlib_crc32(const void* v, size_t len, const TypeDescri
}

case TYPE_DATEV2: {
auto* date_v2_val = (const vectorized::DateV2Value<doris::vectorized::DateV2ValueType>*)v;
char buf[64];
int date_v2_len = date_v2_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, date_v2_len, seed);
return HashUtil::zlib_crc_hash(v, 4, seed);
}

case TYPE_DATETIMEV2: {
auto* date_v2_val =
(const vectorized::DateV2Value<doris::vectorized::DateTimeV2ValueType>*)v;
char buf[64];
int date_v2_len = date_v2_val->to_buffer(buf);
return HashUtil::zlib_crc_hash(buf, date_v2_len, seed);
return HashUtil::zlib_crc_hash(v, 8, seed);
}

case TYPE_DECIMALV2: {
Expand Down
52 changes: 43 additions & 9 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View file
Open in desktop
Expand Up @@ -486,9 +486,9 @@ public Object getRealValue() {
return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second;
} else if (type.equals(Type.DATEV2)) {
return (year << 9) | (month << 5) | day;
} else if (type.equals(Type.DATETIMEV2)) {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
} else if (type.isDatetimeV2()) {
return (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
} else {
Preconditions.checkState(false, "invalid date type: " + type);
return -1L;
Expand All @@ -505,8 +505,8 @@ public ByteBuffer getHashValue(PrimitiveType type) {
buffer.order(ByteOrder.LITTLE_ENDIAN);
buffer.putInt(value);
} else if (type == PrimitiveType.DATETIMEV2) {
long value = (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
long value = (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
buffer = ByteBuffer.allocate(8);
buffer.order(ByteOrder.LITTLE_ENDIAN);
buffer.putLong(value);
Expand Down Expand Up @@ -672,9 +672,40 @@ private long makePackedDatetime() {
return packedDatetime;
}

private void fromPackedDatetimeV2(long packedTime) {
microsecond = (packedTime % (1L << 20));
long ymdhms = (packedTime >> 20);
long ymd = ymdhms >> 17;
day = ymd % (1 << 5);
long ym = ymd >> 5;
month = ym % (1 << 4);
year = ym >> 4;

long hms = ymdhms % (1 << 17);
second = hms % (1 << 6);
minute = (hms >> 6) % (1 << 6);
hour = (hms >> 12);
// set default date literal type to DATETIME
// date literal read from meta will set type by flag bit;
this.type = Type.DATETIMEV2;
}

private void fromPackedDateV2(long packedTime) {
day = packedTime % (1 << 5);
long ym = packedTime >> 5;
month = ym % (1 << 4);
year = ym >> 4;

this.type = Type.DATEV2;
}

private long makePackedDatetimeV2() {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
return (year << 46) | (month << 42) | (day << 37) | (hour << 32)
| (minute << 26) | (second << 20) | (microsecond % (1 << 20));
}

private long makePackedDateV2() {
return ((year << 9) | (month << 5) | day);
}

@Override
Expand All @@ -693,7 +724,7 @@ public void write(DataOutput out) throws IOException {
out.writeInt(((ScalarType) this.type).getScalarScale());
} else if (this.type.equals(Type.DATEV2)) {
out.writeShort(DateLiteralType.DATEV2.value());
out.writeLong(makePackedDatetimeV2());
out.writeLong(makePackedDateV2());
} else {
throw new IOException("Error date literal type : " + type);
}
Expand Down Expand Up @@ -721,14 +752,17 @@ private void fromPackedDatetime(long packedTime) {
public void readFields(DataInput in) throws IOException {
super.readFields(in);
short dateLiteralType = in.readShort();
fromPackedDatetime(in.readLong());
if (dateLiteralType == DateLiteralType.DATETIME.value()) {
fromPackedDatetime(in.readLong());
this.type = Type.DATETIME;
} else if (dateLiteralType == DateLiteralType.DATE.value()) {
fromPackedDatetime(in.readLong());
this.type = Type.DATE;
} else if (dateLiteralType == DateLiteralType.DATETIMEV2.value()) {
fromPackedDatetimeV2(in.readLong());
this.type = ScalarType.createDatetimeV2Type(in.readInt());
} else if (dateLiteralType == DateLiteralType.DATEV2.value()) {
fromPackedDateV2(in.readLong());
this.type = Type.DATEV2;
} else {
throw new IOException("Error date literal type : " + type);
Expand Down
12 changes: 6 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/PartitionKey.java
View file
Open in desktop
Expand Up @@ -59,7 +59,7 @@ public static PartitionKey createInfinityPartitionKey(List<Column> columns, bool
throws AnalysisException {
PartitionKey partitionKey = new PartitionKey();
for (Column column : columns) {
partitionKey.keys.add(LiteralExpr.createInfinity(Type.fromPrimitiveType(column.getDataType()), isMax));
partitionKey.keys.add(LiteralExpr.createInfinity(column.getType(), isMax));
partitionKey.types.add(column.getDataType());
}
return partitionKey;
Expand All @@ -71,15 +71,13 @@ public static PartitionKey createPartitionKey(List<PartitionValue> keys, List<Co
Preconditions.checkArgument(keys.size() <= columns.size());
int i;
for (i = 0; i < keys.size(); ++i) {
partitionKey.keys.add(keys.get(i).getValue(
Type.fromPrimitiveType(columns.get(i).getDataType())));
partitionKey.keys.add(keys.get(i).getValue(columns.get(i).getType()));
partitionKey.types.add(columns.get(i).getDataType());
}

// fill the vacancy with MIN
for (; i < columns.size(); ++i) {
Type type = Type.fromPrimitiveType(columns.get(i).getDataType());
partitionKey.keys.add(LiteralExpr.createInfinity(type, false));
partitionKey.keys.add(LiteralExpr.createInfinity(columns.get(i).getType(), false));
partitionKey.types.add(columns.get(i).getDataType());
}

Expand Down Expand Up @@ -337,7 +335,9 @@ public void readFields(DataInput in) throws IOException {
throw new IOException("type[" + type.name() + "] not supported: ");
}
}
literal.setType(Type.fromPrimitiveType(type));
if (type != PrimitiveType.DATETIMEV2) {
literal.setType(Type.fromPrimitiveType(type));
}
keys.add(literal);
}
}
Expand Down
6 changes: 3 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/load/DeleteHandler.java
View file
Open in desktop
Expand Up @@ -714,7 +714,7 @@ private void checkDeleteV2(OlapTable table, List<Partition> partitions,
binaryPredicate.setChild(1, LiteralExpr.create(value,
ScalarType.createDatetimeV2Type(ScalarType.MAX_DATETIMEV2_SCALE)));
}
LiteralExpr.create(value, Type.fromPrimitiveType(column.getDataType()));
LiteralExpr.create(value, column.getType());
} catch (AnalysisException e) {
// ErrorReport.reportDdlException(ErrorCode.ERR_INVALID_VALUE, value);
throw new DdlException("Invalid column value[" + value + "] for column " + columnName);
Expand All @@ -730,10 +730,10 @@ private void checkDeleteV2(OlapTable table, List<Partition> partitions,
|| column.getDataType() == PrimitiveType.DATEV2
|| column.getDataType() == PrimitiveType.DATETIMEV2) {
DateLiteral dateLiteral = new DateLiteral(value,
Type.fromPrimitiveType(column.getDataType()));
column.getType());
value = dateLiteral.getStringValue();
inPredicate.setChild(i, LiteralExpr.create(value,
Type.fromPrimitiveType(column.getDataType())));
column.getType()));
} else {
LiteralExpr.create(value,
Type.fromPrimitiveType(column.getDataType()));
Expand Down
16 changes: 8 additions & 8 deletions regression-test/data/delete_p0/test_delete.out
View file
Open in desktop
Expand Up @@ -14,29 +14,29 @@
-- !sql --
8

-- !sql --
-- !sql1 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql2 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql3 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111
abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111

-- !sql --
-- !sql4 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql5 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql6 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

-- !sql --
-- !sql7 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111
abcdef 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111 2022-08-16 2022-08-16T11:11:11 2022-08-16T11:11:11.111

-- !sql --
-- !sql8 --
abcdef 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111 2022-08-12 2022-08-16T12:11:11 2022-08-16T12:11:11.111

39 changes: 39 additions & 0 deletions regression-test/data/partition_p0/test_datev2_partition.out
View file
Open in desktop
@@ -0,0 +1,39 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !select --
2022-12-14
2022-12-15
2022-12-16
2022-12-17
2022-12-18
2022-12-19
2022-12-20

-- !select --
2022-12-15

-- !select --
2022-12-16
2022-12-17
2022-12-18
2022-12-19
2022-12-20

-- !select --
2022-12-14T22:22:22.222
2022-12-15T22:22:22.222
2022-12-16T22:22:22.222
2022-12-17T22:22:22.222
2022-12-18T22:22:22.222
2022-12-19T22:22:22.222
2022-12-20T22:22:22.222

-- !select --
2022-12-15T22:22:22.222

-- !select --
2022-12-16T22:22:22.222
2022-12-17T22:22:22.222
2022-12-18T22:22:22.222
2022-12-19T22:22:22.222
2022-12-20T22:22:22.222

22 changes: 11 additions & 11 deletions regression-test/suites/delete_p0/test_delete.groovy
View file
Open in desktop
Expand Up @@ -39,36 +39,36 @@ suite("test_delete") {

sql """ INSERT INTO delete_regression_test VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where k2 = '2022-08-16' """
qt_sql """select * from ${tableName};"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """
qt_sql1 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from delete_regression_test where k3 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """
sql """ delete from ${tableName} where k3 = '2022-08-16 11:11:11' """
qt_sql2 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql3 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where k4 = '2022-08-16 11:11:11.111' """
qt_sql """select * from ${tableName};"""
qt_sql4 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v1 = '2022-08-16' """
qt_sql """select * from ${tableName};"""
qt_sql5 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v2 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql6 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ INSERT INTO ${tableName} VALUES ('abcdef','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111','2022-08-16','2022-08-16 11:11:11.111111','2022-08-16 11:11:11.111111'),('abcdef','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111','2022-08-12','2022-08-16 12:11:11.111111','2022-08-16 12:11:11.111111'); """
sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11' """
qt_sql """select * from ${tableName};"""
qt_sql7 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from ${tableName} where v3 = '2022-08-16 11:11:11.111' """
qt_sql """select * from ${tableName};"""
qt_sql8 """select * from ${tableName} ORDER BY k2;"""
sql """ delete from delete_regression_test where k1 = 'abcdef' """

sql """ DROP TABLE IF EXISTS ${tableName} """
Expand Down
76 changes: 76 additions & 0 deletions regression-test/suites/partition_p0/test_datev2_partition.groovy
View file
Open in desktop
@@ -0,0 +1,76 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_datev2_partition") {
def tblName1 = "test_datev2_partition1"
sql "drop table if exists ${tblName1}"
sql """
CREATE TABLE `${tblName1}` (
`TIME_STAMP` datev2 NOT NULL COMMENT '采集日期'
) ENGINE=OLAP
DUPLICATE KEY(`TIME_STAMP`)
COMMENT 'OLAP'
PARTITION BY RANGE(`TIME_STAMP`)
(PARTITION p20221214 VALUES [('2022-12-14'), ('2022-12-15')),
PARTITION p20221215 VALUES [('2022-12-15'), ('2022-12-16')),
PARTITION p20221216 VALUES [('2022-12-16'), ('2022-12-17')),
PARTITION p20221217 VALUES [('2022-12-17'), ('2022-12-18')),
PARTITION p20221218 VALUES [('2022-12-18'), ('2022-12-19')),
PARTITION p20221219 VALUES [('2022-12-19'), ('2022-12-20')),
PARTITION p20221220 VALUES [('2022-12-20'), ('2022-12-21')),
PARTITION p20221221 VALUES [('2022-12-21'), ('2022-12-22')))
DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
sql """ insert into ${tblName1} values ('2022-12-14'), ('2022-12-15'), ('2022-12-16'), ('2022-12-17'), ('2022-12-18'), ('2022-12-19'), ('2022-12-20') """

qt_select """ select * from ${tblName1} order by TIME_STAMP """
qt_select """ select * from ${tblName1} WHERE TIME_STAMP = '2022-12-15' order by TIME_STAMP """
qt_select """ select * from ${tblName1} WHERE TIME_STAMP > '2022-12-15' order by TIME_STAMP """
sql "drop table ${tblName1}"

def tblName2 = "test_datev2_partition2"
sql "drop table if exists ${tblName2}"
sql """
CREATE TABLE `${tblName2}` (
`TIME_STAMP` datetimev2(3) NOT NULL COMMENT '采集日期'
) ENGINE=OLAP
DUPLICATE KEY(`TIME_STAMP`)
COMMENT 'OLAP'
PARTITION BY RANGE(`TIME_STAMP`)
(PARTITION p20221214 VALUES [('2022-12-14 11:11:11.111'), ('2022-12-15 11:11:11.111')),
PARTITION p20221215 VALUES [('2022-12-15 11:11:11.111'), ('2022-12-16 11:11:11.111')),
PARTITION p20221216 VALUES [('2022-12-16 11:11:11.111'), ('2022-12-17 11:11:11.111')),
PARTITION p20221217 VALUES [('2022-12-17 11:11:11.111'), ('2022-12-18 11:11:11.111')),
PARTITION p20221218 VALUES [('2022-12-18 11:11:11.111'), ('2022-12-19 11:11:11.111')),
PARTITION p20221219 VALUES [('2022-12-19 11:11:11.111'), ('2022-12-20 11:11:11.111')),
PARTITION p20221220 VALUES [('2022-12-20 11:11:11.111'), ('2022-12-21 11:11:11.111')),
PARTITION p20221221 VALUES [('2022-12-21 11:11:11.111'), ('2022-12-22 11:11:11.111')))
DISTRIBUTED BY HASH(`TIME_STAMP`) BUCKETS 10
PROPERTIES (
"replication_allocation" = "tag.location.default: 1"
);
"""
sql """ insert into ${tblName2} values ('2022-12-14 22:22:22.222'), ('2022-12-15 22:22:22.222'), ('2022-12-16 22:22:22.222'), ('2022-12-17 22:22:22.222'), ('2022-12-18 22:22:22.222'), ('2022-12-19 22:22:22.222'), ('2022-12-20 22:22:22.222') """

qt_select """ select * from ${tblName2} order by TIME_STAMP """
qt_select """ select * from ${tblName2} WHERE TIME_STAMP = '2022-12-15 22:22:22.222' order by TIME_STAMP """
qt_select """ select * from ${tblName2} WHERE TIME_STAMP > '2022-12-15 22:22:22.222' order by TIME_STAMP """
sql "drop table ${tblName2}"
}