Skip to content

[Enhancement]: Compatibility_mysql now, curtime, current_time, current_timestamp, localtime, localtimestamp, utc_timestamp #10688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
stalary wants to merge 3 commits into from
Closed

[Enhancement]: Compatibility_mysql now, curtime, current_time, current_timestamp, localtime, localtimestamp, utc_timestamp #10688

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
862f21d
MOD: compatibility_mysql
stalary Jul 7, 2022
f392a28
MOD: mod function in fe and be
stalary Jul 8, 2022
e076fe8
MOD: need be support datetimev2
stalary Jul 10, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions be/src/exprs/timestamp_functions.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -842,6 +842,18 @@ DateTimeVal TimestampFunctions::utc_timestamp(FunctionContext* context) {
return return_val;
}

DateTimeVal TimestampFunctions::now(FunctionContext* context, const IntVal& scale) {
DateTimeValue dtv;
if (!dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000,
context->impl()->state()->timezone_obj())) {
return DateTimeVal::null();
}

DateTimeVal return_val;
dtv.to_datetime_val(&return_val);
return return_val;
}

DateTimeVal TimestampFunctions::now(FunctionContext* context) {
DateTimeValue dtv;
if (!dtv.from_unixtime(context->impl()->state()->timestamp_ms() / 1000,
Expand Down
1 change: 1 addition & 0 deletions be/src/exprs/timestamp_functions.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,7 @@ class TimestampFunctions {
static doris_udf::DateTimeVal timestamp_time_op(doris_udf::FunctionContext* ctx,
const doris_udf::DateTimeVal& ts_val,
const doris_udf::IntVal& count, bool is_add);
static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context, const doris_udf::IntVal& scale);
static doris_udf::DateTimeVal now(doris_udf::FunctionContext* context);
static doris_udf::DoubleVal curtime(doris_udf::FunctionContext* context);
static doris_udf::DateTimeVal curdate(doris_udf::FunctionContext* context);
Expand Down
119 changes: 74 additions & 45 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,14 +150,13 @@ public class DateLiteral extends LiteralExpr {
private static final Pattern HAS_TIME_PART = Pattern.compile("^.*[HhIiklrSsTp]+.*$");

//Date Literal persist type in meta
private enum DateLiteralType {
DATETIME(0),
DATE(1),
private enum DateLiteralType {
DATETIME(0), DATE(1),

DATETIMEV2(2),
DATEV2(3);
DATETIMEV2(2), DATEV2(3);

private final int value;

private DateLiteralType(int value) {
this.value = value;
}
Expand Down Expand Up @@ -244,8 +243,9 @@ public DateLiteral(long year, long month, long day, Type type) {
this.year = year;
this.month = month;
this.day = day;
Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATE.getPrimitiveType())
|| type.getPrimitiveType().equals(Type.DATEV2.getPrimitiveType()));
Preconditions.checkArgument(
type.getPrimitiveType().equals(Type.DATE.getPrimitiveType()) || type.getPrimitiveType()
.equals(Type.DATEV2.getPrimitiveType()));
this.type = type;
}

Expand Down Expand Up @@ -281,8 +281,9 @@ public DateLiteral(long year, long month, long day, long hour, long minute, long
this.year = year;
this.month = month;
this.day = day;
Preconditions.checkArgument(type.getPrimitiveType().equals(Type.DATETIME.getPrimitiveType())
|| type.getPrimitiveType().equals(Type.DATETIMEV2.getPrimitiveType()));
Preconditions.checkArgument(
type.getPrimitiveType().equals(Type.DATETIME.getPrimitiveType()) || type.getPrimitiveType()
.equals(Type.DATETIMEV2.getPrimitiveType()));
this.type = type;
}

Expand All @@ -293,6 +294,35 @@ public DateLiteral(LocalDateTime dateTime, Type type) {
this.hour = dateTime.getHourOfDay();
this.minute = dateTime.getMinuteOfHour();
this.second = dateTime.getSecondOfMinute();
this.microsecond = dateTime.getMillisOfSecond() * 1000L;
this.type = type;
}

public DateLiteral(LocalDateTime dateTime, Type type, int precision) {
this.year = dateTime.getYear();
this.month = dateTime.getMonthOfYear();
this.day = dateTime.getDayOfMonth();
this.hour = dateTime.getHourOfDay();
this.minute = dateTime.getMinuteOfHour();
this.second = dateTime.getSecondOfMinute();
switch (precision) {
case 1:
this.microsecond = dateTime.getMillisOfSecond() / 100 * 100L * 1000L;
break;
case 2:
this.microsecond = dateTime.getMillisOfSecond() / 10 * 10L * 1000L;
break;
case 3:
case 4:
case 5:
case 6:
this.microsecond = dateTime.getMillisOfSecond() * 1000L;
break;
case 0:
default:
this.microsecond = 0;

}
this.type = type;
}

Expand All @@ -301,10 +331,10 @@ public DateLiteral(DateLiteral other) {
hour = other.hour;
minute = other.minute;
second = other.second;
microsecond = other.microsecond;
year = other.year;
month = other.month;
day = other.day;
microsecond = other.microsecond;
type = other.type;
}

Expand Down Expand Up @@ -346,6 +376,7 @@ private void init(String s, Type type) throws AnalysisException {
hour = dateTime.getHourOfDay();
minute = dateTime.getMinuteOfHour();
second = dateTime.getSecondOfMinute();
microsecond = dateTime.getMillisOfSecond() * 1000L;
this.type = type;
} catch (Exception ex) {
throw new AnalysisException("date literal [" + s + "] is invalid");
Expand Down Expand Up @@ -391,8 +422,8 @@ public Object getRealValue() {
} else if (type.equals(Type.DATEV2)) {
return (year << 16) | (month << 8) | day;
} else if (type.equals(Type.DATETIMEV2)) {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
return (year << 50) | (month << 46) | (day << 41) | (hour << 36) | (minute << 30) | (second << 24)
| microsecond;
} else {
Preconditions.checkState(false, "invalid date type: " + type);
return -1L;
Expand Down Expand Up @@ -441,18 +472,22 @@ public String getStringValue() {
if (((ScalarType) type).decimalScale() == 0) {
return s;
}
return s + "." + microsecond / (10L * (6 - ((ScalarType) type).decimalScale()));
return s + "." + getDecimalNumber();
} else {
return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
}
}

public long getDecimalNumber() {
return Double.valueOf(microsecond / (Math.pow(10, 6 - ((ScalarType) type).decimalScale()))).longValue();
}

private String convertToString(PrimitiveType type) {
if (type == PrimitiveType.DATE || type == PrimitiveType.DATEV2) {
return String.format("%04d-%02d-%02d", year, month, day);
} else if (type == PrimitiveType.DATETIMEV2) {
return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d",
year, month, day, hour, minute, second, microsecond);
return String.format("%04d-%02d-%02d %02d:%02d:%02d.%06d", year, month, day, hour, minute, second,
microsecond);
} else {
return String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second);
}
Expand Down Expand Up @@ -518,8 +553,8 @@ private long makePackedDatetime() {
}

private long makePackedDatetimeV2() {
return (year << 50) | (month << 46) | (day << 41) | (hour << 36)
| (minute << 30) | (second << 24) | microsecond;
return (year << 50) | (month << 46) | (day << 41) | (hour << 36) | (minute << 30) | (second << 24)
| microsecond;
}

@Override
Expand Down Expand Up @@ -614,12 +649,8 @@ public long unixTimestamp(TimeZone timeZone) {
public static DateLiteral dateParser(String date, String pattern) throws AnalysisException {
DateTimeFormatter formatter = formatBuilder(pattern).toFormatter();
LocalDateTime dateTime = formatter.parseLocalDateTime(date);
DateLiteral dateLiteral = new DateLiteral(
dateTime.getYear(),
dateTime.getMonthOfYear(),
dateTime.getDayOfMonth(),
dateTime.getHourOfDay(),
dateTime.getMinuteOfHour(),
DateLiteral dateLiteral = new DateLiteral(dateTime.getYear(), dateTime.getMonthOfYear(),
dateTime.getDayOfMonth(), dateTime.getHourOfDay(), dateTime.getMinuteOfHour(),
dateTime.getSecondOfMinute());
if (HAS_TIME_PART.matcher(pattern).matches()) {
dateLiteral.setType(Type.DATETIME);
Expand All @@ -637,8 +668,7 @@ public static boolean hasTimePart(String format) {
//eg : "%Y-%m-%d" or "%Y-%m-%d %H:%i:%s"
public String dateFormat(String pattern) throws AnalysisException {
if (type.equals(Type.DATE)) {
return DATE_FORMATTER.parseLocalDateTime(getStringValue())
.toString(formatBuilder(pattern).toFormatter());
return DATE_FORMATTER.parseLocalDateTime(getStringValue()).toString(formatBuilder(pattern).toFormatter());
} else {
return DATE_TIME_FORMATTER.parseLocalDateTime(getStringValue())
.toString(formatBuilder(pattern).toFormatter());
Expand Down Expand Up @@ -696,23 +726,15 @@ private static DateTimeFormatterBuilder formatBuilder(String pattern) throws Ana
builder.appendHalfdayOfDayText();
break;
case 'r': // %r Time, 12-hour (hh:mm:ss followed by AM or PM)
builder.appendClockhourOfHalfday(2)
.appendLiteral(':')
.appendMinuteOfHour(2)
.appendLiteral(':')
.appendSecondOfMinute(2)
.appendLiteral(' ')
.appendHalfdayOfDayText();
builder.appendClockhourOfHalfday(2).appendLiteral(':').appendMinuteOfHour(2).appendLiteral(':')
.appendSecondOfMinute(2).appendLiteral(' ').appendHalfdayOfDayText();
break;
case 'S': // %S Seconds (00..59)
case 's': // %s Seconds (00..59)
builder.appendSecondOfMinute(2);
break;
case 'T': // %T Time, 24-hour (hh:mm:ss)
builder.appendHourOfDay(2)
.appendLiteral(':')
.appendMinuteOfHour(2)
.appendLiteral(':')
builder.appendHourOfDay(2).appendLiteral(':').appendMinuteOfHour(2).appendLiteral(':')
.appendSecondOfMinute(2);
break;
case 'v': // %v Week (01..53), where Monday is the first day of the week; used with %x
Expand Down Expand Up @@ -741,8 +763,8 @@ private static DateTimeFormatterBuilder formatBuilder(String pattern) throws Ana
case 'X': // %X Year for the week where Sunday is the first day of the week,
// numeric, four digits; used with %V
case 'D': // %D Day of the month with English suffix (0th, 1st, 2nd, 3rd, …)
throw new AnalysisException(String.format("%%%s not supported in date format string",
character));
throw new AnalysisException(
String.format("%%%s not supported in date format string", character));
case '%': // %% A literal "%" character
builder.appendLiteral('%');
break;
Expand Down Expand Up @@ -1082,8 +1104,8 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr
}
} else if (format.charAt(fp) != ' ') {
if (format.charAt(fp) != value.charAt(vp)) {
throw new InvalidFormatException("Invalid char: " + value.charAt(vp) + ", expected: "
+ format.charAt(fp));
throw new InvalidFormatException(
"Invalid char: " + value.charAt(vp) + ", expected: " + format.charAt(fp));
}
fp++;
vp++;
Expand Down Expand Up @@ -1142,9 +1164,8 @@ public int fromDateFormatStr(String format, String value, boolean hasSubVal) thr
// weekday
if (weekNum >= 0 && weekday > 0) {
// Check
if ((strictWeekNumber && (strictWeekNumberYear < 0
|| strictWeekNumberYearType != sundayFirst))
|| (!strictWeekNumber && strictWeekNumberYear >= 0)) {
if ((strictWeekNumber && (strictWeekNumberYear < 0 || strictWeekNumberYearType != sundayFirst)) || (
!strictWeekNumber && strictWeekNumberYear >= 0)) {
throw new InvalidFormatException("invalid week number");
}
long days = calcDaynr(strictWeekNumber ? strictWeekNumberYear : this.year, 1, 1);
Expand Down Expand Up @@ -1349,8 +1370,8 @@ public void fromDateStr(String dateStr) throws AnalysisException {
int start = pre;
int tempVal = 0;
boolean scanToDelim = (!isIntervalFormat) && (fieldIdx != 6);
while (pre < dateStr.length() && Character.isDigit(dateStr.charAt(pre))
&& (scanToDelim || fieldLen-- != 0)) {
while (pre < dateStr.length() && Character.isDigit(dateStr.charAt(pre)) && (scanToDelim
|| fieldLen-- != 0)) {
tempVal = tempVal * 10 + (dateStr.charAt(pre++) - '0');
}
dateVal[fieldIdx] = tempVal;
Expand Down Expand Up @@ -1454,15 +1475,23 @@ public static Type getDefaultDateType(Type type) throws AnalysisException {
}
case DATETIME:
if (Config.use_date_v2_by_default) {
checkScale(type);
return Type.DATETIMEV2;
} else {
return Type.DATETIME;
}
case DATEV2:
case DATETIMEV2:
checkScale(type);
return type;
default:
throw new AnalysisException("Invalid date type: " + type);
}
}

private static void checkScale(Type type) throws AnalysisException {
if (type.getDecimalDigits() < 0 || type.getDecimalDigits() > 6) {
throw new AnalysisException("Invalid decimal scale, type: " + type);
}
}
}
8 changes: 8 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.InvalidFormatException;
Expand Down Expand Up @@ -238,6 +239,13 @@ public static StringLiteral fromUnixTime(LiteralExpr unixTime, StringLiteral fmt
return new StringLiteral(dl.dateFormat(fmtLiteral.getStringValue()));
}

@FEFunction(name = "now", argTypes = { "INT" }, returnType = "DATETIME")
public static DateLiteral nowTinyInt(LiteralExpr scale) throws AnalysisException {
int intScale = (int) scale.getLongValue();
return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())),
DateLiteral.getDefaultDateType(ScalarType.createDatetimeV2Type(intScale)), intScale);
}

@FEFunction(name = "now", argTypes = {}, returnType = "DATETIME")
public static DateLiteral now() throws AnalysisException {
return new DateLiteral(LocalDateTime.now(DateTimeZone.forTimeZone(TimeUtils.getTimeZone())),
Expand Down
16 changes: 8 additions & 8 deletions fe/fe-core/src/test/java/org/apache/doris/rewrite/FEFunctionsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.util.TimeUtils;

import mockit.Expectations;
import mockit.Mocked;
import org.joda.time.DateTime;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Rule;
Expand Down Expand Up @@ -737,13 +737,13 @@ public void timeDiffTest2() throws AnalysisException {

@Test
public void timeNowTest() throws AnalysisException {
String curTimeString = FEFunctions.curTime().toSqlImpl().replace("'", "");
String currentTimestampString = FEFunctions.currentTimestamp().toSqlImpl().replace("'", "");

String nowTimestampString = new DateTime().toString("yyyy-MM-dd HH:mm:ss");
Assert.assertTrue(nowTimestampString.compareTo(currentTimestampString) >= 0);
String nowTimeString = nowTimestampString.substring(nowTimestampString.indexOf(" ") + 1);
Assert.assertTrue(nowTimeString.compareTo(curTimeString) >= 0);
DateLiteral now3 = FEFunctions.nowTinyInt(new IntLiteral(3));
Assert.assertTrue(now3.getDecimalNumber() >= 100 && now3.getDecimalNumber() < 1000);
DateLiteral now5 = FEFunctions.nowTinyInt(new IntLiteral(5));
Assert.assertTrue(now5.getDecimalNumber() >= 10000 && now3.getDecimalNumber() < 100000);
ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
"Invalid decimal scale, type: Datetime(7)",
() -> FEFunctions.nowTinyInt(new IntLiteral(7)));
}

@Test
Expand Down
3 changes: 3 additions & 0 deletions gensrc/script/doris_builtins_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,9 @@
[['now', 'current_timestamp', 'localtime', 'localtimestamp'], 'DATETIME', [],
'_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextE',
'', '', 'vec', 'ALWAYS_NOT_NULLABLE'],
[['now'], 'DATETIME', ['INT'],
'_ZN5doris18TimestampFunctions3nowEPN9doris_udf15FunctionContextERKNS1_6IntValE',
'', '', 'vec', 'ALWAYS_NOT_NULLABLE'],
[['curtime', 'current_time'], 'TIME', [],
'_ZN5doris18TimestampFunctions7curtimeEPN9doris_udf15FunctionContextE',
'', '', 'vec', 'ALWAYS_NOT_NULLABLE'],
Expand Down