Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.apache.doris.nereids.types.DateType;
import org.apache.doris.nereids.types.DateV2Type;
import org.apache.doris.nereids.types.DecimalV3Type;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.TypeCoercionUtils;

import com.google.common.collect.Lists;
Expand Down Expand Up @@ -257,14 +258,15 @@ private Pair<Boolean, Pair<Integer, Integer>> doMatchTypes(FunctionSignature sig
// we need to try to do string literal coercion when search signature.
// for example, FUNC_A has two signature FUNC_A(datetime) and FUNC_A(string)
// if SQL block is `FUNC_A('2020-02-02 00:00:00')`, we should return signature FUNC_A(datetime).
if (!argument.isNullLiteral() && argument.isLiteral() && realType.isStringLikeType()) {
realType = TypeCoercionUtils.characterLiteralTypeCoercion(((Literal) argument).getStringValue(),
Optional<Literal> literalAfterUnwrapNullable = ExpressionUtils.getLiteralAfterUnwrapNullable(argument);
if (!argument.isNullLiteral() && literalAfterUnwrapNullable.isPresent() && realType.isStringLikeType()) {
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

这个得优化器看下

String literalValue = literalAfterUnwrapNullable.get().getStringValue();
realType = TypeCoercionUtils.characterLiteralTypeCoercion(literalValue,
sigArgType).orElse(argument).getDataType();
if (!realType.isStringLikeType()) {
stringLiteralCoersionCount++;
}

String literalValue = ((Literal) argument).getStringValue();
if (sigArgType.isTimeStampTzType()) {
boolean hasTimeZone = DateTimeChecker.hasTimeZone(literalValue);
if (hasTimeZone) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.apache.doris.nereids.trees.expressions.ExecFunction;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
Expand All @@ -29,8 +30,6 @@
import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;

import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;

/**
Expand Down Expand Up @@ -879,26 +878,31 @@ public static Expression milliSecondsSub(TimestampTzLiteral date, BigIntLiteral
*/
@ExecFunction(name = "datediff")
public static Expression dateDiff(DateV2Literal date1, DateV2Literal date2) {
return new IntegerLiteral(dateDiff(date1.toJavaDateType(), date2.toJavaDateType()));
return dateDiff((DateLiteral) date1, date2);
}

@ExecFunction(name = "datediff")
public static Expression dateDiff(DateV2Literal date1, DateTimeV2Literal date2) {
return new IntegerLiteral(dateDiff(date1.toJavaDateType(), date2.toJavaDateType()));
return dateDiff((DateLiteral) date1, date2);
}

@ExecFunction(name = "datediff")
public static Expression dateDiff(DateTimeV2Literal date1, DateV2Literal date2) {
return new IntegerLiteral(dateDiff(date1.toJavaDateType(), date2.toJavaDateType()));
return dateDiff((DateLiteral) date1, date2);
}

@ExecFunction(name = "datediff")
public static Expression dateDiff(DateTimeV2Literal date1, DateTimeV2Literal date2) {
return new IntegerLiteral(dateDiff(date1.toJavaDateType(), date2.toJavaDateType()));
return dateDiff((DateLiteral) date1, date2);
}

private static int dateDiff(LocalDateTime date1, LocalDateTime date2) {
return ((int) ChronoUnit.DAYS.between(date2.toLocalDate(), date1.toLocalDate()));
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

以前的错误在哪?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ChronoUnit 认为0000 年 2 月是闰年,但是我们内部的行为应该是认为0000年是平年, 所以常量折叠如果覆盖到了 0000.2, 就会多一天出来

@ExecFunction(name = "datediff")
public static Expression dateDiff(TimestampTzLiteral date1, TimestampTzLiteral date2) {
return dateDiff((DateLiteral) date1, date2);
}

private static Expression dateDiff(DateLiteral date1, DateLiteral date2) {
return new IntegerLiteral((int) DateV2Literal.dateDiffInDays(date1, date2));
}

@ExecFunction(name = "to_days")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.doris.nereids.trees.expressions.functions.scalar.FromSecond;
import org.apache.doris.nereids.trees.expressions.literal.BigIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal;
Expand All @@ -38,6 +39,7 @@
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TimeV2Literal;
import org.apache.doris.nereids.trees.expressions.literal.TimestampTzLiteral;
import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral;
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
import org.apache.doris.nereids.types.DateTimeV2Type;
Expand Down Expand Up @@ -1109,119 +1111,167 @@ private static Expression fromMicroSecond(long microSecond, int scale) {

@ExecFunction(name = "microseconds_diff")
public static Expression microsecondsDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MICROS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInMicroSeconds(t1, t2));
}

@ExecFunction(name = "microseconds_diff")
public static Expression microsecondsDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInMicroSeconds(t1, t2));
}

@ExecFunction(name = "milliseconds_diff")
public static Expression millisecondsDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MILLIS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInMicroSeconds(t1, t2) / 1000L);
}

@ExecFunction(name = "milliseconds_diff")
public static Expression millisecondsDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInMicroSeconds(t1, t2) / 1000L);
}

@ExecFunction(name = "seconds_diff")
public static Expression secondsDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.SECONDS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2));
}

@ExecFunction(name = "seconds_diff")
public static Expression secondsDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2));
}

@ExecFunction(name = "seconds_diff")
public static Expression secondsDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.SECONDS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2));
}

@ExecFunction(name = "seconds_diff")
public static Expression secondsDiff(DateV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.SECONDS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2));
}

@ExecFunction(name = "seconds_diff")
public static Expression secondsDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.SECONDS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2));
}

@ExecFunction(name = "minutes_diff")
public static Expression minutesDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MINUTES.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L);
}

@ExecFunction(name = "minutes_diff")
public static Expression minutesDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L);
}

@ExecFunction(name = "minutes_diff")
public static Expression minutesDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MINUTES.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L);
}

@ExecFunction(name = "minutes_diff")
public static Expression minutesDiff(DateV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MINUTES.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L);
}

@ExecFunction(name = "minutes_diff")
public static Expression minutesDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MINUTES.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L);
}

@ExecFunction(name = "hours_diff")
public static Expression hoursDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.HOURS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L / 60L);
}

@ExecFunction(name = "hours_diff")
public static Expression hoursDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L / 60L);
}

@ExecFunction(name = "hours_diff")
public static Expression hoursDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.HOURS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L / 60L);
}

@ExecFunction(name = "hours_diff")
public static Expression hoursDiff(DateV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.HOURS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L / 60L);
}

@ExecFunction(name = "hours_diff")
public static Expression hoursDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.HOURS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return new BigIntLiteral(DateTimeV2Literal.datetimeDiffInSecondsRoundToZeroByMicroSecond(t1, t2) / 60L / 60L);
}
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This changes days_diff folding for DateTimeV2 from whole-day duration semantics to calendar-day semantics, which no longer matches BE runtime. BE evaluates days_diff via datetime_diff<DAY> and date_diff_in_days_round_to_zero_by_time, so days_diff('2021-01-02 00:00:00', '2021-01-01 23:59:59') returns 0 at runtime because the difference is less than one full day. With this new toDays(t1) - toDays(t2) folding it returns 1, so enabling constant folding changes query results. The same issue applies to the overloads below where either argument is DateTimeV2Literal; only the pure DateV2 case is safe to compute from day numbers.


@ExecFunction(name = "days_diff")
public static Expression daysDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.DAYS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return daysDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "days_diff")
public static Expression daysDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return daysDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "days_diff")
public static Expression daysDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.DAYS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return daysDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "days_diff")
public static Expression daysDiff(DateV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.DAYS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return daysDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "days_diff")
public static Expression daysDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.DAYS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return daysDiff((DateLiteral) t1, t2);
}

private static Expression daysDiff(DateLiteral t1, DateLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.dateDiffInDaysRoundToZeroByTime(t1, t2));
}

@ExecFunction(name = "weeks_diff")
public static Expression weeksDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.WEEKS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return weeksDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "weeks_diff")
public static Expression weeksDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return weeksDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "weeks_diff")
public static Expression weeksDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.WEEKS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return weeksDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "weeks_diff")
public static Expression weeksDiff(DateV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.WEEKS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return weeksDiff((DateLiteral) t1, t2);
}

@ExecFunction(name = "weeks_diff")
public static Expression weeksDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.WEEKS.between(t2.toJavaDateType(), t1.toJavaDateType()));
return weeksDiff((DateLiteral) t1, t2);
}

private static Expression weeksDiff(DateLiteral t1, DateLiteral t2) {
return new BigIntLiteral(DateTimeV2Literal.dateDiffInDaysRoundToZeroByTime(t1, t2) / 7L);
}

@ExecFunction(name = "months_diff")
public static Expression monthsDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()));
}

@ExecFunction(name = "months_diff")
public static Expression monthsDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()));
}

@ExecFunction(name = "months_diff")
public static Expression monthsDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()));
Expand All @@ -1242,6 +1292,11 @@ public static Expression quartersDiff(DateTimeV2Literal t1, DateTimeV2Literal t2
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()) / 3);
}

@ExecFunction(name = "quarters_diff")
public static Expression quartersDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()) / 3);
}

@ExecFunction(name = "quarters_diff")
public static Expression quartersDiff(DateV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.MONTHS.between(t2.toJavaDateType(), t1.toJavaDateType()) / 3);
Expand All @@ -1252,6 +1307,11 @@ public static Expression yearsDiff(DateTimeV2Literal t1, DateTimeV2Literal t2) {
return new BigIntLiteral(ChronoUnit.YEARS.between(t2.toJavaDateType(), t1.toJavaDateType()));
}

@ExecFunction(name = "years_diff")
public static Expression yearsDiff(TimestampTzLiteral t1, TimestampTzLiteral t2) {
return new BigIntLiteral(ChronoUnit.YEARS.between(t2.toJavaDateType(), t1.toJavaDateType()));
}

@ExecFunction(name = "years_diff")
public static Expression yearsDiff(DateTimeV2Literal t1, DateV2Literal t2) {
return new BigIntLiteral(ChronoUnit.YEARS.between(t2.toJavaDateType(), t1.toJavaDateType()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,10 @@ public Long getValue() {
return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second;
}

public long timePartToMicroSecond() {
return ((hour * 60L + minute) * 60L + second) * 1000L * 1000L + microSecond;
}

@Override
public double getDouble() {
return (double) getValue();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,36 @@ public DateTimeV2Literal(DateTimeV2Type dateType,
roundMicroSecond(dateType.getScale());
}

/** Date difference rounded toward zero by time part. */
public static long dateDiffInDaysRoundToZeroByTime(DateLiteral lhs, DateLiteral rhs) {
long days = DateV2Literal.dateDiffInDays(lhs, rhs);
long microSecondDiff = timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs);
if (days > 0 && microSecondDiff < 0) {
days--;
} else if (days < 0 && microSecondDiff > 0) {
days++;
}
return days;
}

/** Datetime difference in seconds rounded toward zero by microsecond part. */
public static long datetimeDiffInSecondsRoundToZeroByMicroSecond(DateLiteral lhs, DateLiteral rhs) {
return datetimeDiffInMicroSeconds(lhs, rhs) / 1000L / 1000L;
}

/** Datetime difference in microseconds. */
public static long datetimeDiffInMicroSeconds(DateLiteral lhs, DateLiteral rhs) {
return DateV2Literal.dateDiffInDays(lhs, rhs) * 24L * 60L * 60L * 1000L * 1000L
+ timePartToMicroSecond(lhs) - timePartToMicroSecond(rhs);
}

private static long timePartToMicroSecond(DateLiteral date) {
if (date instanceof DateTimeLiteral) {
return ((DateTimeLiteral) date).timePartToMicroSecond();
}
return 0;
}

@Override
public DateTimeV2Type getDataType() throws UnboundException {
return (DateTimeV2Type) super.getDataType();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ public DateV2Literal(long year, long month, long day) {
super(DateV2Type.INSTANCE, year, month, day);
}

public static long dateDiffInDays(DateLiteral lhs, DateLiteral rhs) {
return lhs.getTotalDays() - rhs.getTotalDays();
}

@Override
public org.apache.doris.analysis.DateLiteral toLegacyLiteral() {
return legacyLiteral.get();
Expand Down
Loading
Loading