Skip to content

Commit

Permalink
Settings: Remove mapping.date.round_ceil setting for date math parsing
Browse files Browse the repository at this point in the history 
The setting `mapping.date.round_ceil` (and the undocumented setting
`index.mapping.date.parse_upper_inclusive`) affect how date ranges using
`lte` are parsed.  In elastic#8556 the semantics of date rounding were
solidified, eliminating the need to have different parsing functions
whether the date is inclusive or exclusive.

This change removes these legacy settings and improves the tests
for the date math parser (now at 100% coverage!). It also removes the
unnecessary function `DateMathParser.parseTimeZone` for which
the existing `DateTimeZone.forID` handles all use cases.

Any user previously using these settings can refer to the changed
semantics and change their query accordingly. This is a breaking change
because even dates without datemath previously used the different
parsing functions depending on context.

closes elastic#8598
closes elastic#8889
  • Loading branch information
@rjernst
rjernst committed Dec 15, 2014
1 parent ceafde4 commit 3728728
Show file tree
Hide file tree
Showing 19 changed files with 171 additions and 249 deletions.
16 changes: 9 additions & 7 deletions docs/reference/mapping/date-format.asciidoc
View file
Expand Up @@ -35,13 +35,15 @@ then follow by a math expression, supporting `+`, `-` and `/`
Here are some samples: `now+1h`, `now+1h+1m`, `now+1h/d`,
`2012-01-01||+1M/d`.

Note, when doing `range` type searches, and the upper value is
inclusive, the rounding will properly be rounded to the ceiling instead
of flooring it.

To change this behavior, set
`"mapping.date.round_ceil": false`.

When doing `range` type searches with rounding, the value parsed
depends on whether the end of the range is inclusive or exclusive, and
whether the beginning or end of the range. Rounding up moves to the
last millisecond of the rounding scope, and rounding down to the
first millisecond of the rounding scope. The semantics work as follows:
* `gt` - round up, and use > that value (`2014-11-18||/M` becomes `2014-11-30T23:59:59.999`, ie excluding the entire month)
* `gte` - round D down, and use >= that value (`2014-11-18||/M` becomes `2014-11-01`, ie including the entire month)
* `lt` - round D down, and use < that value (`2014-11-18||/M` becomes `2014-11-01`, ie excluding the entire month)
* `lte` - round D up, and use <= that value(`2014-11-18||/M` becomes `2014-11-30T23:59:59.999`, ie including the entire month)

[float]
[[built-in]]
Expand Down
121 changes: 32 additions & 89 deletions src/main/java/org/elasticsearch/common/joda/DateMathParser.java
View file
Expand Up @@ -19,23 +19,29 @@

package org.elasticsearch.common.joda;

import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.ElasticsearchParseException;
import org.joda.time.DateTimeZone;
import org.joda.time.MutableDateTime;
import org.joda.time.format.DateTimeFormatter;

import java.io.IOException;
import java.util.concurrent.TimeUnit;

/**
* A parser for date/time formatted text with optional date math.
*
* The format of the datetime is configurable, and unix timestamps can also be used. Datemath
* is appended to a datetime with the following syntax:
* <code>||[+-/](\d+)?[yMwdhHms]</code>.
*/
public class DateMathParser {

private final FormatDateTimeFormatter dateTimeFormatter;

private final TimeUnit timeUnit;

public DateMathParser(FormatDateTimeFormatter dateTimeFormatter, TimeUnit timeUnit) {
if (dateTimeFormatter == null) throw new NullPointerException();
if (timeUnit == null) throw new NullPointerException();
this.dateTimeFormatter = dateTimeFormatter;
this.timeUnit = timeUnit;
}
Expand All @@ -44,34 +50,25 @@ public long parse(String text, long now) {
return parse(text, now, false, null);
}

public long parse(String text, long now, boolean roundCeil, DateTimeZone timeZone) {
public long parse(String text, long now, boolean roundUp, DateTimeZone timeZone) {
long time;
String mathString;
if (text.startsWith("now")) {
time = now;
mathString = text.substring("now".length());
} else {
int index = text.indexOf("||");
String parseString;
if (index == -1) {
parseString = text;
mathString = ""; // nothing else
} else {
parseString = text.substring(0, index);
mathString = text.substring(index + 2);
return parseDateTime(text, timeZone);
}
if (roundCeil) {
time = parseRoundCeilStringValue(parseString, timeZone);
} else {
time = parseStringValue(parseString, timeZone);
time = parseDateTime(text.substring(0, index), timeZone);
mathString = text.substring(index + 2);
if (mathString.isEmpty()) {
return time;
}
}

if (mathString.isEmpty()) {
return time;
}

return parseMath(mathString, time, roundCeil);
return parseMath(mathString, time, roundUp);
}

private long parseMath(String mathString, long time, boolean roundUp) throws ElasticsearchParseException {
Expand Down Expand Up @@ -174,7 +171,9 @@ private long parseMath(String mathString, long time, boolean roundUp) throws Ela
}
if (propertyToRound != null) {
if (roundUp) {
propertyToRound.roundCeiling();
// we want to go up to the next whole value, even if we are already on a rounded value
propertyToRound.add(1);
propertyToRound.roundFloor();
dateTime.addMillis(-1); // subtract 1 millisecond to get the largest inclusive value
} else {
propertyToRound.roundFloor();
Expand All @@ -184,83 +183,27 @@ private long parseMath(String mathString, long time, boolean roundUp) throws Ela
return dateTime.getMillis();
}

/**
* Get a DateTimeFormatter parser applying timezone if any.
*/
public static DateTimeFormatter getDateTimeFormatterParser(FormatDateTimeFormatter dateTimeFormatter, DateTimeZone timeZone) {
if (dateTimeFormatter == null) {
return null;
private long parseDateTime(String value, DateTimeZone timeZone) {

// first check for timestamp
if (value.length() > 4 && StringUtils.isNumeric(value)) {
try {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e) {
throw new ElasticsearchParseException("failed to parse date field [" + value + "] as timestamp", e);
}
}

DateTimeFormatter parser = dateTimeFormatter.parser();
if (timeZone != null) {
parser = parser.withZone(timeZone);
}
return parser;
}

private long parseStringValue(String value, DateTimeZone timeZone) {
try {
DateTimeFormatter parser = getDateTimeFormatterParser(dateTimeFormatter, timeZone);
return parser.parseMillis(value);
} catch (RuntimeException e) {
try {
// When date is given as a numeric value, it's a date in ms since epoch
// By definition, it's a UTC date.
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
}
}
}

private long parseRoundCeilStringValue(String value, DateTimeZone timeZone) {
try {
// we create a date time for inclusive upper range, we "include" by default the day level data
// so something like 2011-01-01 will include the full first day of 2011.
// we also use 1970-01-01 as the base for it so we can handle searches like 10:12:55 (just time)
// since when we index those, the base is 1970-01-01
MutableDateTime dateTime = new MutableDateTime(1970, 1, 1, 23, 59, 59, 999, DateTimeZone.UTC);
DateTimeFormatter parser = getDateTimeFormatterParser(dateTimeFormatter, timeZone);
int location = parser.parseInto(dateTime, value, 0);
// if we parsed all the string value, we are good
if (location == value.length()) {
return dateTime.getMillis();
}
// if we did not manage to parse, or the year is really high year which is unreasonable
// see if its a number
if (location <= 0 || dateTime.getYear() > 5000) {
try {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number");
}
}
return dateTime.getMillis();
} catch (RuntimeException e) {
try {
long time = Long.parseLong(value);
return timeUnit.toMillis(time);
} catch (NumberFormatException e1) {
throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
}
}
}

public static DateTimeZone parseZone(String text) throws IOException {
int index = text.indexOf(':');
if (index != -1) {
int beginIndex = text.charAt(0) == '+' ? 1 : 0;
// format like -02:30
return DateTimeZone.forOffsetHoursMinutes(
Integer.parseInt(text.substring(beginIndex, index)),
Integer.parseInt(text.substring(index + 1))
);
} else {
// id, listed here: http://joda-time.sourceforge.net/timezones.html
return DateTimeZone.forID(text);
} catch (IllegalArgumentException e) {

throw new ElasticsearchParseException("failed to parse date field [" + value + "] with format [" + dateTimeFormatter.format() + "]", e);
}
}

Expand Down
26 changes: 3 additions & 23 deletions src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View file
Expand Up @@ -71,9 +71,6 @@
import static org.elasticsearch.index.mapper.core.TypeParsers.parseDateTimeFormatter;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField;

/**
*
*/
public class DateFieldMapper extends NumberFieldMapper<Long> {

public static final String CONTENT_TYPE = "date";
Expand All @@ -90,7 +87,6 @@ public static class Defaults extends NumberFieldMapper.Defaults {
public static final String NULL_VALUE = null;

public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final boolean ROUND_CEIL = true;
}

public static class Builder extends NumberFieldMapper.Builder<Builder, DateFieldMapper> {
Expand Down Expand Up @@ -127,17 +123,12 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {

@Override
public DateFieldMapper build(BuilderContext context) {
boolean roundCeil = Defaults.ROUND_CEIL;
if (context.indexSettings() != null) {
Settings settings = context.indexSettings();
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
if (!locale.equals(dateTimeFormatter.locale())) {
dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale);
}
DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
fieldType.numericPrecisionStep(), boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), coerce(context),
fieldType.numericPrecisionStep(), boost, fieldType, docValues, nullValue, timeUnit, ignoreMalformed(context), coerce(context),
postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(),
multiFieldsBuilder.build(this, context), copyTo);
fieldMapper.includeInAll(includeInAll);
Expand Down Expand Up @@ -182,23 +173,14 @@ public static class TypeParser implements Mapper.TypeParser {

protected FormatDateTimeFormatter dateTimeFormatter;

// Triggers rounding up of the upper bound for range queries and filters if
// set to true.
// Rounding up a date here has the following meaning: If a date is not
// defined with full precision, for example, no milliseconds given, the date
// will be filled up to the next larger date with that precision.
// Example: An upper bound given as "2000-01-01", will be converted to
// "2000-01-01T23.59.59.999"
private final boolean roundCeil;

private final DateMathParser dateMathParser;

private String nullValue;

protected final TimeUnit timeUnit;

protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
String nullValue, TimeUnit timeUnit, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity,

Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
Expand All @@ -208,7 +190,6 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter
this.dateTimeFormatter = dateTimeFormatter;
this.nullValue = nullValue;
this.timeUnit = timeUnit;
this.roundCeil = roundCeil;
this.dateMathParser = new DateMathParser(dateTimeFormatter, timeUnit);
}

Expand Down Expand Up @@ -328,8 +309,7 @@ public long parseToMilliseconds(String value, boolean inclusive, @Nullable DateT
if (forcedDateParser != null) {
dateParser = forcedDateParser;
}
boolean roundUp = inclusive && roundCeil;
return dateParser.parse(value, now, roundUp, zone);
return dateParser.parse(value, now, inclusive, zone);
}

@Override
Expand Down
15 changes: 4 additions & 11 deletions src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Expand Up @@ -49,8 +49,6 @@
import static org.elasticsearch.index.mapper.core.TypeParsers.parseDateTimeFormatter;
import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;

/**
*/
public class TimestampFieldMapper extends DateFieldMapper implements InternalMapper, RootMapper {

public static final String NAME = "_timestamp";
Expand Down Expand Up @@ -123,12 +121,7 @@ public TimestampFieldMapper build(BuilderContext context) {
assert fieldType.stored();
fieldType.setStored(false);
}
boolean roundCeil = Defaults.ROUND_CEIL;
if (context.indexSettings() != null) {
Settings settings = context.indexSettings();
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
Expand Down Expand Up @@ -173,18 +166,18 @@ private static FieldType defaultFieldType(Settings settings) {

public TimestampFieldMapper(Settings indexSettings) {
this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
}

protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Defaults.PRECISION_STEP_64_BIT, Defaults.BOOST, fieldType, docValues,
Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
roundCeil, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings,
ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings,
indexSettings, MultiFields.empty(), null);
this.enabledState = enabledState;
this.path = path;
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
View file
Expand Up @@ -29,7 +29,6 @@
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.joda.DateMathParser;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.common.settings.Settings;
Expand All @@ -38,6 +37,7 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.query.support.QueryParsers;
import org.joda.time.DateTimeZone;

import java.io.IOException;
import java.util.Locale;
Expand Down Expand Up @@ -197,7 +197,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
qpSettings.locale(LocaleUtils.parse(localeStr));
} else if ("time_zone".equals(currentFieldName)) {
try {
qpSettings.timeZone(DateMathParser.parseZone(parser.text()));
qpSettings.timeZone(DateTimeZone.forID(parser.text()));
} catch (IllegalArgumentException e) {
throw new QueryParsingException(parseContext.index(), "[query_string] time_zone [" + parser.text() + "] is unknown");
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/query/RangeFilterParser.java
View file
Expand Up @@ -101,7 +101,7 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
to = parser.objectBytes();
includeUpper = true;
} else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
timeZone = DateMathParser.parseZone(parser.text());
timeZone = DateTimeZone.forID(parser.text());
} else if ("format".equals(currentFieldName)) {
forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
} else {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/elasticsearch/index/query/RangeQueryParser.java
View file
Expand Up @@ -102,7 +102,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
to = parser.objectBytes();
includeUpper = true;
} else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
timeZone = DateMathParser.parseZone(parser.text());
timeZone = DateTimeZone.forID(parser.text());
} else if ("_name".equals(currentFieldName)) {
queryName = parser.text();
} else if ("format".equals(currentFieldName)) {
Expand Down

0 comments on commit 3728728

Please sign in to comment.