Skip to content

Commit

Permalink
Make range queries round up upper bounds again.
Browse files Browse the repository at this point in the history 
Elasticsearch 1.x used to implicitly round up upper bounds of queries when they
were inclusive so that eg. `[2016-09-18 TO 2016-09-20]` would actually run
`[2016-09-18T00:00:00.000Z TO 2016-09-20T23:59:59.999Z]` and include dates like
`2016-09-20T15:32:44`. This behaviour was lost in the cleanups of elastic#8889.

Closes elastic#20579
  • Loading branch information
@jpountz
jpountz committed Oct 7, 2016
1 parent 194a6b1 commit 0c62aa1
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 25 deletions.
27 changes: 19 additions & 8 deletions core/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
View file
Expand Up @@ -63,13 +63,10 @@ public long parse(String text, LongSupplier now, boolean roundUp, DateTimeZone t
} else {
int index = text.indexOf("||");
if (index == -1) {
return parseDateTime(text, timeZone);
return parseDateTime(text, timeZone, roundUp);
}
time = parseDateTime(text.substring(0, index), timeZone);
time = parseDateTime(text.substring(0, index), timeZone, false);
mathString = text.substring(index + 2);
if (mathString.isEmpty()) {
return time;
}
}

return parseMath(mathString, time, roundUp, timeZone);
Expand Down Expand Up @@ -190,15 +187,29 @@ private long parseMath(String mathString, long time, boolean roundUp, DateTimeZo
return dateTime.getMillis();
}

private long parseDateTime(String value, DateTimeZone timeZone) {
private long parseDateTime(String value, DateTimeZone timeZone, boolean roundUpIfNoTime) {
DateTimeFormatter parser = dateTimeFormatter.parser();
if (timeZone != null) {
parser = parser.withZone(timeZone);
}
try {
return parser.parseMillis(value);
MutableDateTime date;
// We use 01/01/1970 as a base date so that things keep working with date
// fields that are filled with times without dates
if (roundUpIfNoTime) {
date = new MutableDateTime(1970, 1, 1, 23, 59, 59, 999, DateTimeZone.UTC);
} else {
date = new MutableDateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
}
final int end = parser.parseInto(date, value, 0);
if (end < 0) {
int position = ~end;
throw new IllegalArgumentException("Parse failure at index [" + position + "] of [" + value + "]");
} else if (end != value.length()) {
throw new IllegalArgumentException("Unrecognized chars at the end of [" + value + "]: [" + value.substring(end) + "]");
}
return date.getMillis();
} catch (IllegalArgumentException e) {

throw new ElasticsearchParseException("failed to parse date field [{}] with format [{}]", e, value, dateTimeFormatter.format());
}
}
Expand Down
48 changes: 45 additions & 3 deletions core/src/test/java/org/elasticsearch/common/joda/DateMathParserTests.java
View file
Expand Up @@ -68,8 +68,14 @@ public void testBasicDates() {
}

public void testRoundingDoesNotAffectExactDate() {
assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00Z", "2014-11-12T22:55:00Z", 0, false, null);
assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00.000Z", "2014-11-12T22:55:00.000Z", 0, false, null);

assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, true, DateTimeZone.forID("+01:00"));
assertDateMathEquals("2014-11-12T22:55:00.000", "2014-11-12T21:55:00.000Z", 0, false, DateTimeZone.forID("+01:00"));

assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, true, null);
assertDateMathEquals("2014-11-12T22:55:00.000+01:00", "2014-11-12T21:55:00.000Z", 0, false, null);
}

public void testTimezone() {
Expand Down Expand Up @@ -134,7 +140,43 @@ public void testNow() {
assertDateMathEquals("now/m", "2014-11-18T14:27", now, false, DateTimeZone.forID("+02:00"));
}

public void testRounding() {
public void testRoundingPreservesEpochAsBaseDate() {
// If a user only specifies times, then the date needs to always be 1970-01-01 regardless of rounding
FormatDateTimeFormatter formatter = Joda.forPattern("HH:mm:ss");
DateMathParser parser = new DateMathParser(formatter);
assertEquals(
this.formatter.parser().parseMillis("1970-01-01T04:52:20.000Z"),
parser.parse("04:52:20", () -> 0, false, null));
assertEquals(
this.formatter.parser().parseMillis("1970-01-01T04:52:20.999Z"),
parser.parse("04:52:20", () -> 0, true, null));
}

// Implicit rounding happening when parts of the date are not specified
public void testImplicitRounding() {
assertDateMathEquals("2014-11-18", "2014-11-18", 0, false, null);
assertDateMathEquals("2014-11-18", "2014-11-18T23:59:59.999Z", 0, true, null);

assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20", 0, false, null);
assertDateMathEquals("2014-11-18T09:20", "2014-11-18T09:20:59.999Z", 0, true, null);

assertDateMathEquals("2014-11-18", "2014-11-17T23:00:00.000Z", 0, false, DateTimeZone.forID("CET"));
assertDateMathEquals("2014-11-18", "2014-11-18T22:59:59.999Z", 0, true, DateTimeZone.forID("CET"));

assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:00.000Z", 0, false, DateTimeZone.forID("CET"));
assertDateMathEquals("2014-11-18T09:20", "2014-11-18T08:20:59.999Z", 0, true, DateTimeZone.forID("CET"));

// implicit rounding with explicit timezone in the date format
FormatDateTimeFormatter formatter = Joda.forPattern("YYYY-MM-ddZ");
DateMathParser parser = new DateMathParser(formatter);
long time = parser.parse("2011-10-09+01:00", () -> 0, false, null);
assertEquals(this.parser.parse("2011-10-09T00:00:00.000+01:00", () -> 0), time);
time = parser.parse("2011-10-09+01:00", () -> 0, true, null);
assertEquals(this.parser.parse("2011-10-09T23:59:59.999+01:00", () -> 0), time);
}

// Explicit rounding using the || separator
public void testExplicitRounding() {
assertDateMathEquals("2014-11-18||/y", "2014-01-01", 0, false, null);
assertDateMathEquals("2014-11-18||/y", "2014-12-31T23:59:59.999", 0, true, null);
assertDateMathEquals("2014||/y", "2014-01-01", 0, false, null);
Expand Down
24 changes: 13 additions & 11 deletions core/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java
View file
Expand Up @@ -36,6 +36,8 @@
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.mapper.DateFieldMapper;
import org.elasticsearch.index.mapper.MappedFieldType;
import org.elasticsearch.index.mapper.DateFieldMapper.DateFieldType;
import org.elasticsearch.index.mapper.MappedFieldType.Relation;
import org.elasticsearch.index.mapper.ParseContext.Document;
Expand Down Expand Up @@ -106,8 +108,8 @@ private void doTestIsFieldWithinQuery(DateFieldType ft, DirectoryReader reader,
public void testIsFieldWithinQuery() throws IOException {
Directory dir = newDirectory();
IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12").getMillis();
long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2016-04-03").getMillis();
Document doc = new Document();
LongPoint field = new LongPoint("my_date", instant1);
doc.add(field);
Expand All @@ -117,7 +119,7 @@ public void testIsFieldWithinQuery() throws IOException {
DirectoryReader reader = DirectoryReader.open(w);
DateFieldType ft = new DateFieldType();
ft.setName("my_date");
DateMathParser alternateFormat = new DateMathParser(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER);
DateMathParser alternateFormat = new DateMathParser(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER);
doTestIsFieldWithinQuery(ft, reader, null, null);
doTestIsFieldWithinQuery(ft, reader, null, alternateFormat);
doTestIsFieldWithinQuery(ft, reader, DateTimeZone.UTC, null);
Expand All @@ -132,7 +134,7 @@ public void testIsFieldWithinQuery() throws IOException {

public void testValueFormat() {
MappedFieldType ft = createDefaultFieldType();
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-12T14:10:55").getMillis();
assertEquals("2015-10-12T14:10:55.000Z",
ft.docValueFormat(null, DateTimeZone.UTC).format(instant));
assertEquals("2015-10-12T15:10:55.000+01:00",
Expand All @@ -141,16 +143,16 @@ public void testValueFormat() {
createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
assertEquals(instant,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
assertEquals(instant,
assertEquals(instant + 999,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
assertEquals(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
}

public void testValueForSearch() {
MappedFieldType ft = createDefaultFieldType();
String date = "2015-10-12T12:09:55.000Z";
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
assertEquals(date, ft.valueForDisplay(instant));
}

Expand All @@ -164,9 +166,9 @@ public void testTermQuery() {
MappedFieldType ft = createDefaultFieldType();
ft.setName("field");
String date = "2015-10-12T14:10:55";
long instant = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
long instant = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date).getMillis();
ft.setIndexOptions(IndexOptions.DOCS);
assertEquals(LongPoint.newExactQuery("field", instant), ft.termQuery(date, context));
assertEquals(LongPoint.newRangeQuery("field", instant, instant + 999), ft.termQuery(date, context));

ft.setIndexOptions(IndexOptions.NONE);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
Expand All @@ -184,8 +186,8 @@ public void testRangeQuery() throws IOException {
ft.setName("field");
String date1 = "2015-10-12T14:10:55";
String date2 = "2016-04-28T11:33:52";
long instant1 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
long instant2 = LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis();
long instant1 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date1).getMillis();
long instant2 = DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parser().parseDateTime(date2).getMillis() + 999;
ft.setIndexOptions(IndexOptions.DOCS);
assertEquals(LongPoint.newRangeQuery("field", instant1, instant2),
ft.rangeQuery(date1, date2, true, true, context).rewrite(new MultiReader()));
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldMapperTests.java
View file
Expand Up @@ -257,7 +257,7 @@ public void testHourFormat() throws Exception {

LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
.rangeQuery("10:00:00", "11:00:00", true, true, context).rewrite(null);
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(11).millis(), DateTimeZone.UTC).getMillis() + 999));
assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(10).millis(), DateTimeZone.UTC).getMillis()));
}

Expand All @@ -284,7 +284,7 @@ public void testDayWithoutYearFormat() throws Exception {

LegacyNumericRangeQuery<Long> rangeQuery = (LegacyNumericRangeQuery<Long>) defaultMapper.mappers().smartNameFieldMapper("date_field").fieldType()
.rangeQuery("Jan 02 10:00:00", "Jan 02 11:00:00", true, true, context).rewrite(null);
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis(), DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMax(), equalTo(new DateTime(TimeValue.timeValueHours(35).millis() + 999, DateTimeZone.UTC).getMillis()));
assertThat(rangeQuery.getMin(), equalTo(new DateTime(TimeValue.timeValueHours(34).millis(), DateTimeZone.UTC).getMillis()));
}

Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/index/mapper/LegacyDateFieldTypeTests.java
View file
Expand Up @@ -138,7 +138,7 @@ public void testValueFormat() {
createDefaultFieldType().docValueFormat("YYYY", DateTimeZone.UTC).format(instant));
assertEquals(instant,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", false, null));
assertEquals(instant,
assertEquals(instant + 999,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12T14:10:55", true, null));
assertEquals(LegacyDateFieldMapper.Defaults.DATE_TIME_FORMATTER.parser().parseDateTime("2015-10-13").getMillis() - 1,
ft.docValueFormat(null, DateTimeZone.UTC).parseLong("2015-10-12||/d", true, null));
Expand Down

0 comments on commit 0c62aa1

Please sign in to comment.