elastic · rjernst · Dec 15, 2014 · Dec 11, 2014
diff --git a/docs/reference/mapping/date-format.asciidoc b/docs/reference/mapping/date-format.asciidoc
@@ -35,13 +35,15 @@ then follow by a math expression, supporting `+`, `-` and `/`
 Here are some samples: `now+1h`, `now+1h+1m`, `now+1h/d`,
 `2012-01-01||+1M/d`.
 
-Note, when doing `range` type searches, and the upper value is
-inclusive, the rounding will properly be rounded to the ceiling instead
-of flooring it.
-
-To change this behavior, set 
-`"mapping.date.round_ceil": false`.
-
+When doing `range` type searches with rounding, the value parsed
+depends on whether the end of the range is inclusive or exclusive, and
+whether the beginning or end of the range.  Rounding up moves to the
+last millisecond of the rounding scope, and rounding down to the
+first millisecond of the rounding scope. The semantics work as follows:
+* `gt` - round up, and use > that value (`2014-11-18||/M` becomes `2014-11-30T23:59:59.999`, ie excluding the entire month)
+* `gte` - round D down, and use >= that value (`2014-11-18||/M` becomes `2014-11-01`, ie including the entire month)
+* `lt` - round D down, and use < that value (`2014-11-18||/M` becomes `2014-11-01`, ie excluding the entire month)
+* `lte` - round D up, and use <= that value(`2014-11-18||/M` becomes `2014-11-30T23:59:59.999`, ie including the entire month)
 
 [float]
 [[built-in]]

diff --git a/src/main/java/org/elasticsearch/common/joda/DateMathParser.java b/src/main/java/org/elasticsearch/common/joda/DateMathParser.java
@@ -19,23 +19,29 @@
 
 package org.elasticsearch.common.joda;
 
+import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
 import org.joda.time.format.DateTimeFormatter;
 
-import java.io.IOException;
 import java.util.concurrent.TimeUnit;
 
 /**
+ * A parser for date/time formatted text with optional date math.
+ * 
+ * The format of the datetime is configurable, and unix timestamps can also be used. Datemath
+ * is appended to a datetime with the following syntax:
+ * <code>||[+-/](\d+)?[yMwdhHms]</code>.
  */
 public class DateMathParser {
 
     private final FormatDateTimeFormatter dateTimeFormatter;
-
     private final TimeUnit timeUnit;
 
     public DateMathParser(FormatDateTimeFormatter dateTimeFormatter, TimeUnit timeUnit) {
+        if (dateTimeFormatter == null) throw new NullPointerException();
+        if (timeUnit == null) throw new NullPointerException();
         this.dateTimeFormatter = dateTimeFormatter;
         this.timeUnit = timeUnit;
     }
@@ -44,34 +50,25 @@ public long parse(String text, long now) {
         return parse(text, now, false, null);
     }
 
-    public long parse(String text, long now, boolean roundCeil, DateTimeZone timeZone) {
+    public long parse(String text, long now, boolean roundUp, DateTimeZone timeZone) {
         long time;
         String mathString;
         if (text.startsWith("now")) {
             time = now;
             mathString = text.substring("now".length());
         } else {
             int index = text.indexOf("||");
-            String parseString;
             if (index == -1) {
-                parseString = text;
-                mathString = ""; // nothing else
-            } else {
-                parseString = text.substring(0, index);
-                mathString = text.substring(index + 2);
+                return parseDateTime(text, timeZone);
             }
-            if (roundCeil) {
-                time = parseRoundCeilStringValue(parseString, timeZone);
-            } else {
-                time = parseStringValue(parseString, timeZone);
+            time = parseDateTime(text.substring(0, index), timeZone);
+            mathString = text.substring(index + 2);
+            if (mathString.isEmpty()) {
+                return time;
             }
         }
 
-        if (mathString.isEmpty()) {
-            return time;
-        }
-
-        return parseMath(mathString, time, roundCeil);
+        return parseMath(mathString, time, roundUp);
     }
 
     private long parseMath(String mathString, long time, boolean roundUp) throws ElasticsearchParseException {
@@ -174,7 +171,9 @@ private long parseMath(String mathString, long time, boolean roundUp) throws Ela
             }
             if (propertyToRound != null) {
                 if (roundUp) {
-                    propertyToRound.roundCeiling();
+                    // we want to go up to the next whole value, even if we are already on a rounded value
+                    propertyToRound.add(1);
+                    propertyToRound.roundFloor();
                     dateTime.addMillis(-1); // subtract 1 millisecond to get the largest inclusive value
                 } else {
                     propertyToRound.roundFloor();
@@ -184,83 +183,27 @@ private long parseMath(String mathString, long time, boolean roundUp) throws Ela
         return dateTime.getMillis();
     }
 
-    /**
-     * Get a DateTimeFormatter parser applying timezone if any.
-     */
-    public static DateTimeFormatter getDateTimeFormatterParser(FormatDateTimeFormatter dateTimeFormatter, DateTimeZone timeZone) {
-        if (dateTimeFormatter == null) {
-            return null;
+    private long parseDateTime(String value, DateTimeZone timeZone) {
+
+        // first check for timestamp
+        if (value.length() > 4 && StringUtils.isNumeric(value)) {
+            try {
+                long time = Long.parseLong(value);
+                return timeUnit.toMillis(time);
+            } catch (NumberFormatException e) {
+                throw new ElasticsearchParseException("failed to parse date field [" + value + "] as timestamp", e);
+            }
         }
-
+        
         DateTimeFormatter parser = dateTimeFormatter.parser();
         if (timeZone != null) {
             parser = parser.withZone(timeZone);
         }
-        return parser;
-    }
-
-    private long parseStringValue(String value, DateTimeZone timeZone) {
         try {
-            DateTimeFormatter parser = getDateTimeFormatterParser(dateTimeFormatter, timeZone);
             return parser.parseMillis(value);
-        } catch (RuntimeException e) {
-            try {
-                // When date is given as a numeric value, it's a date in ms since epoch
-                // By definition, it's a UTC date.
-                long time = Long.parseLong(value);
-                return timeUnit.toMillis(time);
-            } catch (NumberFormatException e1) {
-                throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
-            }
-        }
-    }
-
-    private long parseRoundCeilStringValue(String value, DateTimeZone timeZone) {
-        try {
-            // we create a date time for inclusive upper range, we "include" by default the day level data
-            // so something like 2011-01-01 will include the full first day of 2011.
-            // we also use 1970-01-01 as the base for it so we can handle searches like 10:12:55 (just time)
-            // since when we index those, the base is 1970-01-01
-            MutableDateTime dateTime = new MutableDateTime(1970, 1, 1, 23, 59, 59, 999, DateTimeZone.UTC);
-            DateTimeFormatter parser = getDateTimeFormatterParser(dateTimeFormatter, timeZone);
-            int location = parser.parseInto(dateTime, value, 0);
-            // if we parsed all the string value, we are good
-            if (location == value.length()) {
-                return dateTime.getMillis();
-            }
-            // if we did not manage to parse, or the year is really high year which is unreasonable
-            // see if its a number
-            if (location <= 0 || dateTime.getYear() > 5000) {
-                try {
-                    long time = Long.parseLong(value);
-                    return timeUnit.toMillis(time);
-                } catch (NumberFormatException e1) {
-                    throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number");
-                }
-            }
-            return dateTime.getMillis();
-        } catch (RuntimeException e) {
-            try {
-                long time = Long.parseLong(value);
-                return timeUnit.toMillis(time);
-            } catch (NumberFormatException e1) {
-                throw new ElasticsearchParseException("failed to parse date field [" + value + "], tried both date format [" + dateTimeFormatter.format() + "], and timestamp number", e);
-            }
-        }
-    }
-
-    public static DateTimeZone parseZone(String text) throws IOException {
-        int index = text.indexOf(':');
-        if (index != -1) {
-            int beginIndex = text.charAt(0) == '+' ? 1 : 0;
-            // format like -02:30
-            return DateTimeZone.forOffsetHoursMinutes(
-                    Integer.parseInt(text.substring(beginIndex, index)),
-                    Integer.parseInt(text.substring(index + 1))
-            );
-        } else {
-            // id, listed here: http://joda-time.sourceforge.net/timezones.html
-            return DateTimeZone.forID(text);
+        } catch (IllegalArgumentException e) {
+
+            throw new ElasticsearchParseException("failed to parse date field [" + value + "] with format [" + dateTimeFormatter.format() + "]", e);
         }
     }
 

diff --git a/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
@@ -71,9 +71,6 @@
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseDateTimeFormatter;
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField;
 
-/**
- *
- */
 public class DateFieldMapper extends NumberFieldMapper<Long> {
 
     public static final String CONTENT_TYPE = "date";
@@ -90,7 +87,6 @@ public static class Defaults extends NumberFieldMapper.Defaults {
         public static final String NULL_VALUE = null;
 
         public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
-        public static final boolean ROUND_CEIL = true;
     }
 
     public static class Builder extends NumberFieldMapper.Builder<Builder, DateFieldMapper> {
@@ -127,17 +123,12 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
 
         @Override
         public DateFieldMapper build(BuilderContext context) {
-            boolean roundCeil = Defaults.ROUND_CEIL;
-            if (context.indexSettings() != null) {
-                Settings settings = context.indexSettings();
-                roundCeil =  settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
-            }
             fieldType.setOmitNorms(fieldType.omitNorms() && boost == 1.0f);
             if (!locale.equals(dateTimeFormatter.locale())) {
                 dateTimeFormatter = new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale);
             }
             DateFieldMapper fieldMapper = new DateFieldMapper(buildNames(context), dateTimeFormatter,
-                    fieldType.numericPrecisionStep(), boost, fieldType, docValues, nullValue, timeUnit, roundCeil, ignoreMalformed(context), coerce(context),
+                    fieldType.numericPrecisionStep(), boost, fieldType, docValues, nullValue, timeUnit, ignoreMalformed(context), coerce(context),
                     postingsProvider, docValuesProvider, similarity, normsLoading, fieldDataSettings, context.indexSettings(),
                     multiFieldsBuilder.build(this, context), copyTo);
             fieldMapper.includeInAll(includeInAll);
@@ -182,23 +173,14 @@ public static class TypeParser implements Mapper.TypeParser {
 
     protected FormatDateTimeFormatter dateTimeFormatter;
 
-    // Triggers rounding up of the upper bound for range queries and filters if
-    // set to true.
-    // Rounding up a date here has the following meaning: If a date is not
-    // defined with full precision, for example, no milliseconds given, the date
-    // will be filled up to the next larger date with that precision.
-    // Example: An upper bound given as "2000-01-01", will be converted to
-    // "2000-01-01T23.59.59.999"
-    private final boolean roundCeil;
-
     private final DateMathParser dateMathParser;
 
     private String nullValue;
 
     protected final TimeUnit timeUnit;
 
     protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter, int precisionStep, float boost, FieldType fieldType, Boolean docValues,
-                              String nullValue, TimeUnit timeUnit, boolean roundCeil, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
+                              String nullValue, TimeUnit timeUnit, Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce,
                               PostingsFormatProvider postingsProvider, DocValuesFormatProvider docValuesProvider, SimilarityProvider similarity,
 
                               Loading normsLoading, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, CopyTo copyTo) {
@@ -208,7 +190,6 @@ protected DateFieldMapper(Names names, FormatDateTimeFormatter dateTimeFormatter
         this.dateTimeFormatter = dateTimeFormatter;
         this.nullValue = nullValue;
         this.timeUnit = timeUnit;
-        this.roundCeil = roundCeil;
         this.dateMathParser = new DateMathParser(dateTimeFormatter, timeUnit);
     }
 
@@ -328,8 +309,7 @@ public long parseToMilliseconds(String value, boolean inclusive, @Nullable DateT
         if (forcedDateParser != null) {
             dateParser = forcedDateParser;
         }
-        boolean roundUp = inclusive && roundCeil;
-        return dateParser.parse(value, now, roundUp, zone);
+        return dateParser.parse(value, now, inclusive, zone);
     }
 
     @Override

diff --git a/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java b/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
@@ -49,8 +49,6 @@
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseDateTimeFormatter;
 import static org.elasticsearch.index.mapper.core.TypeParsers.parseField;
 
-/**
- */
 public class TimestampFieldMapper extends DateFieldMapper implements InternalMapper, RootMapper {
 
     public static final String NAME = "_timestamp";
@@ -123,12 +121,7 @@ public TimestampFieldMapper build(BuilderContext context) {
                 assert fieldType.stored();
                 fieldType.setStored(false);
             }
-            boolean roundCeil = Defaults.ROUND_CEIL;
-            if (context.indexSettings() != null) {
-                Settings settings = context.indexSettings();
-                roundCeil =  settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
-            }
-            return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
+            return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp,
                     ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
         }
     }
@@ -173,18 +166,18 @@ private static FieldType defaultFieldType(Settings settings) {
 
     public TimestampFieldMapper(Settings indexSettings) {
         this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
-                Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
+             Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
     }
 
     protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
-                                   FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
+                                   FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp,
                                    Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
                                    DocValuesFormatProvider docValuesProvider, Loading normsLoading,
                                    @Nullable Settings fieldDataSettings, Settings indexSettings) {
         super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
                 Defaults.PRECISION_STEP_64_BIT, Defaults.BOOST, fieldType, docValues,
                 Defaults.NULL_VALUE, TimeUnit.MILLISECONDS /*always milliseconds*/,
-                roundCeil, ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, 
+                ignoreMalformed, coerce, postingsProvider, docValuesProvider, null, normsLoading, fieldDataSettings, 
                 indexSettings, MultiFields.empty(), null);
         this.enabledState = enabledState;
         this.path = path;

diff --git a/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java b/src/main/java/org/elasticsearch/index/query/QueryStringQueryParser.java
@@ -29,7 +29,6 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
-import org.elasticsearch.common.joda.DateMathParser;
 import org.elasticsearch.common.lucene.search.Queries;
 import org.elasticsearch.common.regex.Regex;
 import org.elasticsearch.common.settings.Settings;
@@ -38,6 +37,7 @@
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.support.QueryParsers;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.Locale;
@@ -197,7 +197,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
                     qpSettings.locale(LocaleUtils.parse(localeStr));
                 } else if ("time_zone".equals(currentFieldName)) {
                     try {
-                        qpSettings.timeZone(DateMathParser.parseZone(parser.text()));
+                        qpSettings.timeZone(DateTimeZone.forID(parser.text()));
                     } catch (IllegalArgumentException e) {
                         throw new QueryParsingException(parseContext.index(), "[query_string] time_zone [" + parser.text() + "] is unknown");
                     }

diff --git a/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java b/src/main/java/org/elasticsearch/index/query/RangeFilterParser.java
@@ -101,7 +101,7 @@ public Filter parse(QueryParseContext parseContext) throws IOException, QueryPar
                             to = parser.objectBytes();
                             includeUpper = true;
                         } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
-                            timeZone = DateMathParser.parseZone(parser.text());
+                            timeZone = DateTimeZone.forID(parser.text());
                         } else if ("format".equals(currentFieldName)) {
                             forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
                         } else {

diff --git a/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java b/src/main/java/org/elasticsearch/index/query/RangeQueryParser.java
@@ -102,7 +102,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
                     to = parser.objectBytes();
                     includeUpper = true;
                 } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
-                    timeZone = DateMathParser.parseZone(parser.text());
+                    timeZone = DateTimeZone.forID(parser.text());
                 } else if ("_name".equals(currentFieldName)) {
                     queryName = parser.text();
                 } else if ("format".equals(currentFieldName)) {