Skip to content

Commit

Permalink
Make range queries round up upper bounds again. (#20582)
Browse files Browse the repository at this point in the history 
Elasticsearch 1.x used to implicitly round up upper bounds of queries when they
were inclusive so that eg. `[2016-09-18 TO 2016-09-20]` would actually run
`[2016-09-18T00:00:00.000Z TO 2016-09-20T23:59:59.999Z]` and include dates like
`2016-09-20T15:32:44`. This behaviour was lost in the cleanups of #8889.

Closes #20579
  • Loading branch information
@jpountz
jpountz committed Oct 7, 2016
1 parent d01a629 commit c1e5421
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 25 deletions.
27 changes: 19 additions & 8 deletions core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
View file
Expand Up @@ -63,13 +63,10 @@ public long parse(String text, LongSupplier now, boolean roundUp, DateTimeZone t
} else {
int index = text.indexOf("||");
if (index == -1) {
return parseDateTime(text, timeZone);
return parseDateTime(text, timeZone, roundUp);
}
time = parseDateTime(text.substring(0, index), timeZone);
time = parseDateTime(text.substring(0, index), timeZone, false);
mathString = text.substring(index + 2);
if (mathString.isEmpty()) {
return time;
}
}

return parseMath(mathString, time, roundUp, timeZone);
Expand Down Expand Up @@ -190,15 +187,29 @@ private long parseMath(String mathString, long time, boolean roundUp, DateTimeZo
return dateTime.getMillis();
}

private long parseDateTime(String value, DateTimeZone timeZone) {
private long parseDateTime(String value, DateTimeZone timeZone, boolean roundUpIfNoTime) {
DateTimeFormatter parser = dateTimeFormatter.parser();
if (timeZone != null) {
parser = parser.withZone(timeZone);
}
try {
return parser.parseMillis(value);
MutableDateTime date;
// We use 01/01/1970 as a base date so that things keep working with date
// fields that are filled with times without dates
if (roundUpIfNoTime) {
date = new MutableDateTime(1970, 1, 1, 23, 59, 59, 999, DateTimeZone.UTC);
} else {
date = new MutableDateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
}
final int end = parser.parseInto(date, value, 0);
if (end < 0) {
int position = ~end;
throw new IllegalArgumentException("Parse failure at index [" + position + "] of [" + value + "]");
} else if (end != value.length()) {
throw new IllegalArgumentException("Unrecognized chars at the end of [" + value + "]: [" + value.substring(end) + "]");
}
return date.getMillis();
} catch (IllegalArgumentException e) {

throw new ElasticsearchParseException("failed to parse date field [{}] with format [{}]", e, value, dateTimeFormatter.format());
}
}
Expand Down
48 changes: 45 additions & 3 deletions core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
View file
Expand Up @@ -68,8 +68,14 @@ public void testBasicDates() {
}

public void testRoundingDoesNotAffectExactDate() {
assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, false, null);
assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, false, null);

assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, true, DateTimeZone.forID("+01:00"));
assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, false, DateTimeZone.forID("+01:00"));

assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, false, null);
}

public void testTimezone() {
Expand Down Expand Up @@ -134,7 +140,43 @@ public void testNow() {
assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, DateTimeZone.forID("+02:00"));
}

public void testRounding() {
public void testRoundingPreservesEpochAsBaseDate() {
// If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding
FormatDateTimeFormatter formatter = Joda.forPattern("HH:mm:ss");
DateMathParser parser = new DateMathParser(formatter);
assertEquals(
this.formatter.parser().parseMillis("1970-01-01T04:52:20.000Z"),
parser.parse("04:52:20", () -> 0, false, null));
assertEquals(
this.formatter.parser().parseMillis("1970-01-01T04:52:20.999Z"),
parser.parse("04:52:20", () -> 0, true, null));
}

// Implicit rounding happening when parts of the date are not specified
public void testImplicitRounding() {
assertDateMathEquals("2014-11-18", "2014-11-18", 0, false, null);
assertDateMathEquals("2014-11-18", "2014-11-18T23:59:59.999Z", 0, true, null);

assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20", 0, false, null);
assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20:59.999Z", 0, true, null);

assertDateMathEquals("2014-11-18", "2014-11-17T23:00:00.000Z", 0, false, DateTimeZone.forID("CET"));
assertDateMathEquals("2014-11-18", "2014-11-18T22:59:59.999Z", 0, true, DateTimeZone.forID("CET"));

assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:00.000Z", 0, false, DateTimeZone.forID("CET"));
assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:59.999Z", 0, true, DateTimeZone.forID("CET"));

// implicit rounding with explicit timezone in the date format
FormatDateTimeFormatter formatter = Joda.forPattern("YYYY-MM-ddZ");
DateMathParser parser = new DateMathParser(formatter);
long time = parser.parse("2011-10-09+01:00", () -> 0, false, null);
assertEquals(this.parser.parse("2011-10-09T00:00:00.000+01:00", () -> 0), time);
time = parser.parse("2011-10-09+01:00", () -> 0, true, null);
assertEquals(this.parser.parse("2011-10-09T23:59:59.999+01:00", () -> 0), time);
}

// Explicit rounding using the || separator
public void testExplicitRounding() {
assertDateMathEquals("2014-11-18||/y", "2014-01-01", 0, false, null);
assertDateMathEquals("2014-11-18||/y", "2014-12-31T23:59:59.999", 0, true, null);
assertDateMathEquals("2014||/y", "2014-01-01", 0, false, null);
Expand Down
24 changes: 13 additions & 11 deletions core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
View file
Expand Up @@ -36,6 +36,8 @@
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
import org.elasticsearch.index.mapper.MappedFieldType.Relation;
import org.elasticsearch.index.mapper.ParseContext.Document;
Expand Down Expand Up @@ -106,8 +108,8 @@ private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader,
public void testIsFieldWithinQuery() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
Document doc = new Document();
LongPoint field = new LongPoint("my_date", instant1);
doc.add(field);
Expand All @@ -117,7 +119,7 @@ public void testIsFieldWithinQuery() throws IOException {
DirectoryReader reader = DirectoryReader.open(w);
DateFieldType ft = new DateFieldType();
ft.setName("my_date");
DateMathParser alternateFormat = new DateMathParser(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER);
DateMathParser alternateFormat = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
doTestIsFieldWithinQuery(ft, reader, null, null);
doTestIsFieldWithinQuery(ft, reader, null, alternateFormat);
doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, null);
Expand All @@ -132,7 +134,7 @@ public void testIsFieldWithinQuery() throws IOException {

public void testValueFormat() {
MappedFieldType ft = createDefaultFieldType();
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
assertEquals("2015-10-12T14:10:55.000Z",
ft.docValueFormat(null, DateTimeZone.UTC).format(instant));
assertEquals("2015-10-12T15:10:55.000+01:00",
Expand All @@ -141,16 +143,16 @@ public void testValueFormat() {
createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
assertEquals(instant,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
assertEquals(instant,
assertEquals(instant + 999,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
assertEquals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
}

public void testValueForSearch() {
MappedFieldType ft = createDefaultFieldType();
String date = "2015-10-12T12:09:55.000Z";
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
assertEquals(date, ft.valueForDisplay(instant));
}

Expand All @@ -164,9 +166,9 @@ public void testTermQuery() {
MappedFieldType ft = createDefaultFieldType();
ft.setName("field");
String date = "2015-10-12T14:10:55";
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
ft.setIndexOptions(IndexOptions.DOCS);
assertEquals(LongPoint.newExactQuery("field", instant), ft.termQuery(date, context));
assertEquals(LongPoint.newRangeQuery("field", instant, instant + 999), ft.termQuery(date, context));

ft.setIndexOptions(IndexOptions.NONE);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
Expand All @@ -184,8 +186,8 @@ public void testRangeQuery() throws IOException {
ft.setName("field");
String date1 = "2015-10-12T14:10:55";
String date2 = "2016-04-28T11:33:52";
long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis();
long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis() + 999;
ft.setIndexOptions(IndexOptions.DOCS);
assertEquals(LongPoint.newRangeQuery("field", instant1, instant2),
ft.rangeQuery(date1, date2, true, true, context).rewrite(new MultiReader()));
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java
View file
Expand Up @@ -257,7 +257,7 @@ public void testHourFormat() throws Exception {

LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
.rangeQuery("10:00:00", "11:00:00", true, true, context).rewrite(null);
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis() + 999));
assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(10).millis(), DateTimeZone.UTC).getMillis()));
}

Expand All @@ -284,7 +284,7 @@ public void testDayWithoutYearFormat() throws Exception {

LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
.rangeQuery("Jan 02 10:00:00", "Jan 02 11:00:00", true, true, context).rewrite(null);
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis(), DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis() + 999, DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(34).millis(), DateTimeZone.UTC).getMillis()));
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java
View file
Expand Up @@ -138,7 +138,7 @@ public void testValueFormat() {
createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
assertEquals(instant,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
assertEquals(instant,
assertEquals(instant + 999,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
Expand Down

0 comments on commit c1e5421

Please sign in to comment.