Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CALCITE-6269] Fix missing/broken BigQuery date-time format elements #3761

Merged
merged 1 commit into from
Apr 22, 2024
+511 −304
Merged

[CALCITE-6269] Fix missing/broken BigQuery date-time format elements #3761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 10 additions & 5 deletions core/src/main/java/org/apache/calcite/util/Bug.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -200,16 +200,21 @@ public abstract class Bug {
* Fix to be available with Avatica 1.24.0 [CALCITE-6053] */
public static final boolean CALCITE_6092_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE-6269">[CALCITE-6269]
* Fix missing/broken BigQuery date-time format elements</a> is fixed. */
public static final boolean CALCITE_6269_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE-6270">[CALCITE-6270]
* Support FORMAT in CAST from Numeric and BYTES to String (Enabled in BigQuery)</a> is fixed. */
public static final boolean CALCITE_6270_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE-6367">[CALCITE-6367]
* Add timezone support for FORMAT clause in CAST (enabled in BigQuery)</a> is fixed. */
public static final boolean CALCITE_6367_FIXED = false;

/** Whether
* <a href="https://issues.apache.org/jira/browse/CALCITE-6375">[CALCITE-6375]
* Fix failing quidem tests for FORMAT in CAST</a> is fixed. */
public static final boolean CALCITE_6375_FIXED = false;

/**
* Use this to flag temporary code.
*/
Expand Down
232 changes: 191 additions & 41 deletions core/src/main/java/org/apache/calcite/util/format/FormatElementEnum.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import org.apache.calcite.avatica.util.DateTimeUtils;

import org.apache.commons.lang3.StringUtils;

import org.checkerframework.checker.nullness.qual.Nullable;

import java.text.DateFormat;
Expand Down Expand Up @@ -57,19 +59,22 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%d", calendar.get(Calendar.DAY_OF_WEEK)));
}
},
DAY("EEEE", "The full weekday name") {
DAY("EEEE", "The full weekday name, in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
// The Calendar and SimpleDateFormatter do not seem to give correct results
// for the day of the week prior to the Julian to Gregorian date change.
// So we resort to using a LocalDate representation.
LocalDate ld =
LocalDate.of(calendar.get(Calendar.YEAR),
// Calendar months are numbered from 0
calendar.get(Calendar.MONTH) + 1,
calendar.get(Calendar.DAY_OF_MONTH));
sb.append(ld.getDayOfWeek().getDisplayName(TextStyle.FULL, Locale.ENGLISH));
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.FULL).toUpperCase(Locale.ROOT));
}
},
Day("EEEE", "The full weekday name, capitalized") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.FULL));
}
},
day("EEEE", "The full weekday name, in lowercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.FULL).toLowerCase(Locale.ROOT));
}
},
DD("dd", "The day of the month as a decimal number (01-31)") {
Expand All @@ -86,19 +91,22 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%03d", calendar.get(Calendar.DAY_OF_YEAR)));
}
},
DY("EEE", "The abbreviated weekday name") {
DY("EEE", "The abbreviated weekday name, in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
// The Calendar and SimpleDateFormatter do not seem to give correct results
// for the day of the week prior to the Julian to Gregorian date change.
// So we resort to using a LocalDate representation.
LocalDate ld =
LocalDate.of(calendar.get(Calendar.YEAR),
// Calendar months are numbered from 0
calendar.get(Calendar.MONTH) + 1,
calendar.get(Calendar.DAY_OF_MONTH));
sb.append(ld.getDayOfWeek().getDisplayName(TextStyle.SHORT, Locale.ENGLISH));
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.SHORT).toUpperCase(Locale.ROOT));
}
},
Dy("EEE", "The abbreviated weekday name, capitalized") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.SHORT));
}
},
dy("EEE", "The abbreviated weekday name, in lowercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.getDayFromDate(date, TextStyle.SHORT).toLowerCase(Locale.ROOT));
}
},
E("d", "The day of the month as a decimal number (1-31); "
Expand All @@ -112,37 +120,79 @@ public enum FormatElementEnum implements FormatElement {
FF1("S", "Fractional seconds to 1 digit") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.sFormat.format(date));
// Extracting 1 decimal place as SimpleDateFormat returns precision with 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(work.sssFormat.format(date).charAt(0));
}
},
FF2("SS", "Fractional seconds to 2 digits") {
FF2("S", "Fractional seconds to 2 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.ssFormat.format(date));
// Extracting 2 decimal places as SimpleDateFormat returns precision with 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(work.sssFormat.format(date), 0, 2);
}
},
FF3("SSS", "Fractional seconds to 3 digits") {
FF3("S", "Fractional seconds to 3 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.sssFormat.format(date));
}
},
FF4("SSSS", "Fractional seconds to 4 digits") {
FF4("S", "Fractional seconds to 4 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.ssssFormat.format(date));
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 4, "0"));
}
},
FF5("SSSSS", "Fractional seconds to 5 digits") {
FF5("S", "Fractional seconds to 5 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.sssssFormat.format(date));
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 5, "0"));
}
},
FF6("SSSSSS", "Fractional seconds to 6 digits") {
FF6("S", "Fractional seconds to 6 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.ssssssFormat.format(date));
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 6, "0"));
}
},
FF7("S", "Fractional seconds to 6 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 7, "0"));
}
},
FF8("S", "Fractional seconds to 6 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 8, "0"));
}
},
FF9("S", "Fractional seconds to 6 digits") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
// Padding zeroes to right as SimpleDateFormat supports precision only up to 3 places.
// Refer to <a href="https://issues.apache.org/jira/projects/CALCITE/issues/CALCITE-6269">
// [CALCITE-6269] Fix missing/broken BigQuery date-time format elements</a>.
sb.append(StringUtils.rightPad(work.sssFormat.format(date), 9, "0"));
}
},
HH12("h", "The hour (12-hour clock) as a decimal number (01-12)") {
Expand Down Expand Up @@ -185,18 +235,42 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%02d", calendar.get(Calendar.MONTH) + 1));
}
},
MON("MMM", "The abbreviated month name") {
MON("MMM", "The abbreviated month name, in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmFormat.format(date).toUpperCase(Locale.ROOT));
}
},
Mon("MMM", "The abbreviated month name, capitalized") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmFormat.format(date));
}
},
MONTH("MMMM", "The full month name (English)") {
mon("MMM", "The abbreviated month name, in lowercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmFormat.format(date).toLowerCase(Locale.ROOT));
}
},
MONTH("MMMM", "The full month name (English), in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmmFormat.format(date).toUpperCase(Locale.ROOT));
}
},
Month("MMMM", "The full month name (English), capitalized") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmmFormat.format(date));
}
},
month("MMMM", "The full month name (English), in lowercase") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.mmmmFormat.format(date).toLowerCase(Locale.ROOT));
}
},
// PM can represent both AM and PM
PM("a", "Meridian indicator without periods") {
@Override public void format(StringBuilder sb, Date date) {
Expand All @@ -217,6 +291,34 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%d", (calendar.get(Calendar.MONTH) / 3) + 1));
}
},
AMPM("", "The time as Meridian Indicator in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
sb.append(calendar.get(Calendar.AM_PM) == Calendar.AM ? "AM" : "PM");
Anthrino marked this conversation as resolved.
Show resolved Hide resolved
}
},
AM_PM("", "The time as Meridian Indicator in uppercase with dot") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
sb.append(calendar.get(Calendar.AM_PM) == Calendar.AM ? "A.M." : "P.M.");
}
},
ampm("", "The time as Meridian Indicator in lowercase") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
sb.append(calendar.get(Calendar.AM_PM) == Calendar.AM ? "am" : "pm");
}
},
am_pm("", "The time as Meridian Indicator in uppercase") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
sb.append(calendar.get(Calendar.AM_PM) == Calendar.AM ? "a.m." : "p.m.");
}
},
MS("SSS", "The millisecond as a decimal number (000-999)") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
Expand All @@ -231,6 +333,24 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%02d", calendar.get(Calendar.SECOND)));
}
},
SSSSS("s", "The seconds of the day (00000-86400)") {
@Override public void format(StringBuilder sb, Date date) {
Calendar calendar = Work.get().calendar;
calendar.setTime(date);
long timeInMillis = calendar.getTimeInMillis();

// Set calendar to start of day for input date
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.SECOND, 0);
calendar.set(Calendar.MILLISECOND, 0);
long dayStartInMillis = calendar.getTimeInMillis();

// Get seconds of the day as difference from day start time
long secondsPassed = (timeInMillis - dayStartInMillis) / 1000;
sb.append(String.format(Locale.ROOT, "%05d", secondsPassed));
}
},
TZR("z", "The time zone name") {
@Override public void format(StringBuilder sb, Date date) {
// TODO: how to support timezones?
Expand All @@ -253,13 +373,33 @@ public enum FormatElementEnum implements FormatElement {
sb.append(String.format(Locale.ROOT, "%02d", calendar.get(Calendar.WEEK_OF_YEAR)));
}
},
Y("y", "Last digit of year") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
String formattedYear = work.yyFormat.format(date);
sb.append(formattedYear.substring(formattedYear.length() - 1));
}
},
YY("yy", "Last 2 digits of year") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.yyFormat.format(date));
}
},
YYY("yyy", "Last 3 digits of year") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
String formattedYear = work.yyyyFormat.format(date);
sb.append(formattedYear.substring(formattedYear.length() - 3));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens for a year like 50?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are right, the earlier method was reusing what was being done in YYYY and cropping that down to three digits. But the YYYY implementation was incorrect as it was reading an integer where the Calendar object stored the year, and that would return an int value and not a formatted year with 4 digits (showing the century number).
Corrected the YYYY and YYY functions to return a proper formatted value in terms of digits, and added a method pctY to return values consistent to what BQ returns.

}
},
YYYY("yyyy", "The year with century as a decimal number") {
@Override public void format(StringBuilder sb, Date date) {
final Work work = Work.get();
sb.append(work.yyyyFormat.format(date));
}
},
pctY("yyyy", "The year with century as a decimal number") {
@Override public void format(StringBuilder sb, Date date) {
final Calendar calendar = Work.get().calendar;
calendar.setTime(date);
Expand Down Expand Up @@ -306,12 +446,22 @@ static Work get() {
* https://issues.apache.org/jira/browse/CALCITE-6252. This may be
* specific to Java 11. */
final DateFormat mmmmFormat = new SimpleDateFormat(MONTH.javaFmt, Locale.US);
final DateFormat sFormat = new SimpleDateFormat(FF1.javaFmt, Locale.ROOT);
final DateFormat ssFormat = new SimpleDateFormat(FF2.javaFmt, Locale.ROOT);
final DateFormat sssFormat = new SimpleDateFormat(FF3.javaFmt, Locale.ROOT);
final DateFormat ssssFormat = new SimpleDateFormat(FF4.javaFmt, Locale.ROOT);
final DateFormat sssssFormat = new SimpleDateFormat(FF5.javaFmt, Locale.ROOT);
final DateFormat ssssssFormat = new SimpleDateFormat(FF6.javaFmt, Locale.ROOT);
final DateFormat yyFormat = new SimpleDateFormat(YY.javaFmt, Locale.ROOT);
final DateFormat yyyyFormat = new SimpleDateFormat(YYYY.javaFmt, Locale.ROOT);

/** Util to return the full or abbreviated weekday name from date and expected TextStyle. */
private String getDayFromDate(Date date, TextStyle style) {
calendar.setTime(date);
// The Calendar and SimpleDateFormatter do not seem to give correct results
// for the day of the week prior to the Julian to Gregorian date change.
// So we resort to using a LocalDate representation.
LocalDate ld =
LocalDate.of(calendar.get(Calendar.YEAR),
// Calendar months are numbered from 0
calendar.get(Calendar.MONTH) + 1,
calendar.get(Calendar.DAY_OF_MONTH));
return ld.getDayOfWeek().getDisplayName(style, Locale.ENGLISH);
}
}
}