Skip to content

Commit

Permalink
Support optional parsers in any order with DateMathParser and roundup (
Browse files Browse the repository at this point in the history
…elastic#46654)

Currently DateMathParser with roundUp = true is relying on the DateFormatter build with
combined optional sub parsers with defaulted fields (depending on the formatter).
That means that for yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSS
Java.time implementation expects optional parsers in order from most specific to
least specific (reverse in the example above).
It is causing a problem because the first parsing succeeds but does not consume the full input.
The second parser should be used.
We can work around this with keeping a list of RoundUpParsers and iterate over them choosing
the one that parsed full input. The same approach we used for regular (non date math) in
relates elastic#40100
The jdk is not considering this to be a bug https://bugs.openjdk.java.net/browse/JDK-8188771

Those below will expect this change first
relates elastic#46242
relates elastic#45284
  • Loading branch information
pgomulka committed Sep 27, 2019
1 parent e943e27 commit 3fc200b
Show file tree
Hide file tree
Showing 8 changed files with 147 additions and 58 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ static DateFormatter forPattern(String input) {
return formatters.get(0);
}

return DateFormatters.merge(input, formatters);
return JavaDateFormatter.combined(input, formatters);
}

static List<String> splitCombinedPatterns(String input) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1033,7 +1033,7 @@ public class DateFormatters {
new DateTimeFormatterBuilder().appendValue(WeekFields.ISO.weekBasedYear()).toFormatter(Locale.ROOT));

/*
* Returns a formatter for a four digit weekyear. (uuuu)
* Returns a formatter for a four digit year. (uuuu)
*/
private static final DateFormatter YEAR = new JavaDateFormatter("year",
new DateTimeFormatterBuilder().appendValue(ChronoField.YEAR).toFormatter(Locale.ROOT));
Expand Down Expand Up @@ -1605,27 +1605,6 @@ static DateFormatter forPattern(String input) {
}
}

static JavaDateFormatter merge(String pattern, List<DateFormatter> formatters) {
assert formatters.size() > 0;

List<DateTimeFormatter> dateTimeFormatters = new ArrayList<>(formatters.size());
DateTimeFormatterBuilder roundupBuilder = new DateTimeFormatterBuilder();
DateTimeFormatter printer = null;
for (DateFormatter formatter : formatters) {
assert formatter instanceof JavaDateFormatter;
JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter;
if (printer == null) {
printer = javaDateFormatter.getPrinter();
}
dateTimeFormatters.addAll(javaDateFormatter.getParsers());
roundupBuilder.appendOptional(javaDateFormatter.getRoundupParser());
}
DateTimeFormatter roundUpParser = roundupBuilder.toFormatter(Locale.ROOT);

return new JavaDateFormatter(pattern, printer, builder -> builder.append(roundUpParser),
dateTimeFormatters.toArray(new DateTimeFormatter[0]));
}

private static final LocalDate LOCALDATE_EPOCH = LocalDate.of(1970, 1, 1);

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalField;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -57,19 +58,33 @@ class JavaDateFormatter implements DateFormatter {
private final String format;
private final DateTimeFormatter printer;
private final List<DateTimeFormatter> parsers;
private final DateTimeFormatter roundupParser;
private final JavaDateFormatter roundupParser;

private JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter roundupParser, List<DateTimeFormatter> parsers) {
this.format = format;
this.printer = printer;
this.roundupParser = roundupParser;
this.parsers = parsers;
static class RoundUpFormatter extends JavaDateFormatter{

RoundUpFormatter(String format, List<DateTimeFormatter> roundUpParsers) {
super(format, firstFrom(roundUpParsers),null, roundUpParsers);
}

private static DateTimeFormatter firstFrom(List<DateTimeFormatter> roundUpParsers) {
return roundUpParsers.get(0);
}

@Override
JavaDateFormatter getRoundupParser() {
throw new UnsupportedOperationException("RoundUpFormatter does not have another roundUpFormatter");
}
}

// named formatters use default roundUpParser
JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeFormatter... parsers) {
this(format, printer, builder -> ROUND_UP_BASE_FIELDS.forEach(builder::parseDefaulting), parsers);
}

JavaDateFormatter(String format, DateTimeFormatter printer, Consumer<DateTimeFormatterBuilder> roundupParserConsumer,
// subclasses override roundUpParser
JavaDateFormatter(String format,
DateTimeFormatter printer,
Consumer<DateTimeFormatterBuilder> roundupParserConsumer,
DateTimeFormatter... parsers) {
if (printer == null) {
throw new IllegalArgumentException("printer may not be null");
Expand All @@ -90,20 +105,51 @@ private JavaDateFormatter(String format, DateTimeFormatter printer, DateTimeForm
} else {
this.parsers = Arrays.asList(parsers);
}
//this is when the RoundUp Formatter is created. In further merges (with ||) it will only append this one to a list.
List<DateTimeFormatter> roundUp = createRoundUpParser(format, roundupParserConsumer);
this.roundupParser = new RoundUpFormatter(format, roundUp) ;
}

DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
private List<DateTimeFormatter> createRoundUpParser(String format,
Consumer<DateTimeFormatterBuilder> roundupParserConsumer) {
if (format.contains("||") == false) {
DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder();
builder.append(this.parsers.get(0));
roundupParserConsumer.accept(builder);
return Arrays.asList(builder.toFormatter(locale()));
}
roundupParserConsumer.accept(builder);
DateTimeFormatter roundupFormatter = builder.toFormatter(locale());
if (printer.getZone() != null) {
roundupFormatter = roundupFormatter.withZone(zone());
return null;
}

public static DateFormatter combined(String input, List<DateFormatter> formatters) {
assert formatters.size() > 0;

List<DateTimeFormatter> parsers = new ArrayList<>(formatters.size());
List<DateTimeFormatter> roundUpParsers = new ArrayList<>(formatters.size());

DateTimeFormatter printer = null;
for (DateFormatter formatter : formatters) {
assert formatter instanceof JavaDateFormatter;
JavaDateFormatter javaDateFormatter = (JavaDateFormatter) formatter;
if (printer == null) {
printer = javaDateFormatter.getPrinter();
}
parsers.addAll(javaDateFormatter.getParsers());
roundUpParsers.addAll(javaDateFormatter.getRoundupParser().getParsers());
}
this.roundupParser = roundupFormatter;

return new JavaDateFormatter(input, printer, roundUpParsers, parsers);
}

private JavaDateFormatter(String format, DateTimeFormatter printer, List<DateTimeFormatter> roundUpParsers,
List<DateTimeFormatter> parsers) {
this.format = format;
this.printer = printer;
this.roundupParser = roundUpParsers != null ? new RoundUpFormatter(format, roundUpParsers ) : null;
this.parsers = parsers;
}

DateTimeFormatter getRoundupParser() {
JavaDateFormatter getRoundupParser() {
return roundupParser;
}

Expand Down Expand Up @@ -162,8 +208,12 @@ public DateFormatter withZone(ZoneId zoneId) {
if (zoneId.equals(zone())) {
return this;
}
return new JavaDateFormatter(format, printer.withZone(zoneId), getRoundupParser().withZone(zoneId),
parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList()));
List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withZone(zoneId)).collect(Collectors.toList());
List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
.stream()
.map(p -> p.withZone(zoneId))
.collect(Collectors.toList());
return new JavaDateFormatter(format, printer.withZone(zoneId), roundUpParsers, parsers);
}

@Override
Expand All @@ -172,8 +222,12 @@ public DateFormatter withLocale(Locale locale) {
if (locale.equals(locale())) {
return this;
}
return new JavaDateFormatter(format, printer.withLocale(locale), getRoundupParser().withLocale(locale),
parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList()));
List<DateTimeFormatter> parsers = this.parsers.stream().map(p -> p.withLocale(locale)).collect(Collectors.toList());
List<DateTimeFormatter> roundUpParsers = this.roundupParser.getParsers()
.stream()
.map(p -> p.withLocale(locale))
.collect(Collectors.toList());
return new JavaDateFormatter(format, printer.withLocale(locale), roundUpParsers, parsers);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,12 @@
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalAdjusters;
import java.time.temporal.TemporalQueries;
import java.util.Objects;
import java.util.function.Function;
import java.util.function.LongSupplier;

/**
Expand All @@ -48,14 +46,14 @@
public class JavaDateMathParser implements DateMathParser {

private final JavaDateFormatter formatter;
private final DateTimeFormatter roundUpFormatter;
private final String format;
private final JavaDateFormatter roundupParser;

JavaDateMathParser(String format, JavaDateFormatter formatter, DateTimeFormatter roundUpFormatter) {
JavaDateMathParser(String format, JavaDateFormatter formatter, JavaDateFormatter roundupParser) {
this.format = format;
this.roundupParser = roundupParser;
Objects.requireNonNull(formatter);
this.formatter = formatter;
this.roundUpFormatter = roundUpFormatter;
}

@Override
Expand Down Expand Up @@ -217,12 +215,12 @@ private Instant parseDateTime(String value, ZoneId timeZone, boolean roundUpIfNo
throw new ElasticsearchParseException("cannot parse empty date");
}

Function<String,TemporalAccessor> formatter = roundUpIfNoTime ? this.roundUpFormatter::parse : this.formatter::parse;
DateFormatter formatter = roundUpIfNoTime ? this.roundupParser : this.formatter;
try {
if (timeZone == null) {
return DateFormatters.from(formatter.apply(value)).toInstant();
return DateFormatters.from(formatter.parse(value)).toInstant();
} else {
TemporalAccessor accessor = formatter.apply(value);
TemporalAccessor accessor = formatter.parse(value);
ZoneId zoneId = TemporalQueries.zone().queryFrom(accessor);
if (zoneId != null) {
timeZone = zoneId;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,20 @@

package org.elasticsearch.common.joda;

import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.bootstrap.JavaVersion;
import org.elasticsearch.common.time.DateFormatter;
import org.elasticsearch.common.time.DateFormatters;
import org.elasticsearch.common.time.DateMathParser;
import org.elasticsearch.test.ESTestCase;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.format.ISODateTimeFormat;
import org.joda.time.format.DateTimeFormat;

import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
Expand All @@ -39,6 +44,34 @@
import static org.hamcrest.Matchers.is;

public class JavaJodaTimeDuellingTests extends ESTestCase {
public void testCompositeDateMathParsing(){
//in all these examples the second pattern will be used
assertDateMathEquals("2014-06-06T12:01:02.123", "yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SSS");
assertDateMathEquals("2014-06-06T12:01:02.123", "strictDateTimeNoMillis||yyyy-MM-dd'T'HH:mm:ss.SSS");
assertDateMathEquals("2014-06-06T12:01:02.123", "yyyy-MM-dd'T'HH:mm:ss+HH:MM||yyyy-MM-dd'T'HH:mm:ss.SSS");
}

public void testExceptionWhenCompositeParsingFailsDateMath(){
//both parsing failures should contain pattern and input text in exception
//both patterns fail parsing the input text due to only 2 digits of millis. Hence full text was not parsed.
String pattern = "yyyy-MM-dd'T'HH:mm:ss||yyyy-MM-dd'T'HH:mm:ss.SS";
String text = "2014-06-06T12:01:02.123";
ElasticsearchParseException e1 = expectThrows(ElasticsearchParseException.class,
() -> dateMathToMillis(text, DateFormatter.forPattern(pattern)));
assertThat(e1.getMessage(), containsString(pattern));
assertThat(e1.getMessage(), containsString(text));

ElasticsearchParseException e2 = expectThrows(ElasticsearchParseException.class,
() -> dateMathToMillis(text, Joda.forPattern(pattern)));
assertThat(e2.getMessage(), containsString(pattern));
assertThat(e2.getMessage(), containsString(text));
}

private long dateMathToMillis(String text, DateFormatter dateFormatter) {
DateFormatter javaFormatter = dateFormatter.withLocale(randomLocale(random()));
DateMathParser javaDateMath = javaFormatter.toDateMathParser();
return javaDateMath.parse(text, () -> 0, true, (ZoneId) null).toEpochMilli();
}

//these parsers should allow both ',' and '.' as a decimal point
public void testDecimalPointParsing(){
Expand Down Expand Up @@ -871,4 +904,11 @@ private void assertJavaTimeParseException(String input, String format) {
assertThat(e.getMessage(), containsString(input));
assertThat(e.getMessage(), containsString(format));
}

private void assertDateMathEquals(String text, String pattern) {
long gotMillisJava = dateMathToMillis(text, DateFormatter.forPattern(pattern));
long gotMillisJoda = dateMathToMillis(text, Joda.forPattern(pattern));

assertEquals(gotMillisJoda, gotMillisJava);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@

import java.time.Instant;
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.LongSupplier;

Expand Down Expand Up @@ -59,6 +60,19 @@ void assertDateEquals(long gotMillis, String original, String expected) {
}
}

public void testOverridingLocaleOrZoneAndCompositeRoundUpParser() {
//the pattern has to be composite and the match should not be on the first one
DateFormatter formatter = Joda.forPattern("date||epoch_millis").withLocale(randomLocale(random()));
DateMathParser parser = formatter.toDateMathParser();
long gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli();
assertDateEquals(gotMillis, "297276785531", "297276785531");

formatter = Joda.forPattern("date||epoch_millis").withZone(ZoneOffset.UTC);
parser = formatter.toDateMathParser();
gotMillis = parser.parse("297276785531", () -> 0, true, (ZoneId) null).toEpochMilli();
assertDateEquals(gotMillis, "297276785531", "297276785531");
}

public void testBasicDates() {
assertDateMathEquals("2014", "2014-01-01T00:00:00.000");
assertDateMathEquals("2014-05", "2014-05-01T00:00:00.000");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import java.time.ZoneId;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.util.Locale;
Expand Down Expand Up @@ -253,7 +252,7 @@ public void testIso8601Parsing() {
public void testRoundupFormatterWithEpochDates() {
assertRoundupFormatter("epoch_millis", "1234567890", 1234567890L);
// also check nanos of the epoch_millis formatter if it is rounded up to the nano second
DateTimeFormatter roundUpFormatter = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_millis")).getRoundupParser();
JavaDateFormatter roundUpFormatter = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_millis")).getRoundupParser();
Instant epochMilliInstant = DateFormatters.from(roundUpFormatter.parse("1234567890")).toInstant();
assertThat(epochMilliInstant.getLong(ChronoField.NANO_OF_SECOND), is(890_999_999L));

Expand All @@ -266,7 +265,7 @@ public void testRoundupFormatterWithEpochDates() {

assertRoundupFormatter("epoch_second", "1234567890", 1234567890999L);
// also check nanos of the epoch_millis formatter if it is rounded up to the nano second
DateTimeFormatter epochSecondRoundupParser = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_second")).getRoundupParser();
JavaDateFormatter epochSecondRoundupParser = ((JavaDateFormatter) DateFormatter.forPattern("8epoch_second")).getRoundupParser();
Instant epochSecondInstant = DateFormatters.from(epochSecondRoundupParser.parse("1234567890")).toInstant();
assertThat(epochSecondInstant.getLong(ChronoField.NANO_OF_SECOND), is(999_999_999L));

Expand All @@ -280,7 +279,7 @@ public void testRoundupFormatterWithEpochDates() {
private void assertRoundupFormatter(String format, String input, long expectedMilliSeconds) {
JavaDateFormatter dateFormatter = (JavaDateFormatter) DateFormatter.forPattern(format);
dateFormatter.parse(input);
DateTimeFormatter roundUpFormatter = dateFormatter.getRoundupParser();
JavaDateFormatter roundUpFormatter = dateFormatter.getRoundupParser();
long millis = DateFormatters.from(roundUpFormatter.parse(input)).toInstant().toEpochMilli();
assertThat(millis, is(expectedMilliSeconds));
}
Expand All @@ -290,8 +289,8 @@ public void testRoundupFormatterZone() {
String format = randomFrom("epoch_second", "epoch_millis", "strict_date_optional_time", "uuuu-MM-dd'T'HH:mm:ss.SSS",
"strict_date_optional_time||date_optional_time");
JavaDateFormatter formatter = (JavaDateFormatter) DateFormatter.forPattern(format).withZone(zoneId);
DateTimeFormatter roundUpFormatter = formatter.getRoundupParser();
assertThat(roundUpFormatter.getZone(), is(zoneId));
JavaDateFormatter roundUpFormatter = formatter.getRoundupParser();
assertThat(roundUpFormatter.zone(), is(zoneId));
assertThat(formatter.zone(), is(zoneId));
}

Expand All @@ -300,8 +299,8 @@ public void testRoundupFormatterLocale() {
String format = randomFrom("epoch_second", "epoch_millis", "strict_date_optional_time", "uuuu-MM-dd'T'HH:mm:ss.SSS",
"strict_date_optional_time||date_optional_time");
JavaDateFormatter formatter = (JavaDateFormatter) DateFormatter.forPattern(format).withLocale(locale);
DateTimeFormatter roundupParser = formatter.getRoundupParser();
assertThat(roundupParser.getLocale(), is(locale));
JavaDateFormatter roundupParser = formatter.getRoundupParser();
assertThat(roundupParser.locale(), is(locale));
assertThat(formatter.locale(), is(locale));
}

Expand Down
Loading

0 comments on commit 3fc200b

Please sign in to comment.