Date Parsing: Add parsing for epoch and epoch in milliseconds

This commit changes the date handling. First and foremost Elasticsearch
does not try to convert every date to a unix timestamp first and then
uses the configured date. This now allows for dates like `2015121212` to
be parsed correctly.

Instead it is now explicit by adding a `epoch_second` and `epoch_millis`
date format. This also means, that the default date format now is
`epoch_millis||dateOptionalTime` to remain backwards compatible.

Closes elastic#5328
Relates elastic#10971

docs/reference/mapping/date-format.asciidoc

@@ -198,6 +198,11 @@ year.
 
 |`year_month_day`|A formatter for a four digit year, two digit month of
 year, and two digit day of month.
+
+|`epoch_second`|A formatter for the number of seconds since the epoch.
+
+|`epoch_millis`|A formatter for the number of milliseconds since
+the epoch.
 |=======================================================================
 
 [float]

docs/reference/mapping/fields/timestamp-field.asciidoc

@@ -79,7 +79,7 @@ format>> used to parse the provided timestamp value. For example:
 }
 --------------------------------------------------
 
-Note, the default format is `dateOptionalTime`. The timestamp value will
+Note, the default format is `epoch_millis||dateOptionalTime`. The timestamp value will
 first be parsed as a number and if it fails the format will be tried.
 
 [float]

docs/reference/mapping/types/core-types.asciidoc

@@ -349,7 +349,7 @@ date type:
 Defaults to the property/field name.
 
 |`format` |The <<mapping-date-format,date
-format>>. Defaults to `dateOptionalTime`.
+format>>. Defaults to `epoch_millis||dateOptionalTime`.
 
 |`store` |Set to `true` to store actual field in the index, `false` to not
 store it. Defaults to `false` (note, the JSON document itself is stored,

docs/reference/mapping/types/root-object-type.asciidoc

@@ -42,8 +42,8 @@ and will use the matching format as its format attribute. The date
 format itself is explained
 <<mapping-date-format,here>>.
 
-The default formats are: `dateOptionalTime` (ISO) and
-`yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z`.
+The default formats are: `dateOptionalTime` (ISO),
+`yyyy/MM/dd HH:mm:ss Z||yyyy/MM/dd Z` and `epoch_millis`.
 
 *Note:* `dynamic_date_formats` are used *only* for dynamically added
 date fields, not for `date` fields that you specify in your mapping.

src/main/java/org/elasticsearch/action/TimestampParsingException.java

@@ -32,6 +32,11 @@ public TimestampParsingException(String timestamp) {
         this.timestamp = timestamp;
     }
 
+    public TimestampParsingException(String timestamp, Throwable cause) {
+        super("failed to parse timestamp [" + timestamp + "]", cause);
+        this.timestamp = timestamp;
+    }
+
     public String timestamp() {
         return timestamp;
     }

src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java

@@ -161,19 +161,11 @@ public int hashCode() {
     public static class Timestamp {
 
         public static String parseStringTimestamp(String timestampAsString, FormatDateTimeFormatter dateTimeFormatter) throws TimestampParsingException {
-            long ts;
             try {
-                // if we manage to parse it, its a millisecond timestamp, just return the string as is
-                ts = Long.parseLong(timestampAsString);
-                return timestampAsString;
-            } catch (NumberFormatException e) {
-                try {
-                    ts = dateTimeFormatter.parser().parseMillis(timestampAsString);
-                } catch (RuntimeException e1) {
-                    throw new TimestampParsingException(timestampAsString);
-                }
+                return Long.toString(dateTimeFormatter.parser().parseMillis(timestampAsString));
+            } catch (RuntimeException e) {
+                throw new TimestampParsingException(timestampAsString, e);
             }
-            return Long.toString(ts);
         }
 
 

src/main/java/org/elasticsearch/common/joda/DateMathParser.java

@@ -19,14 +19,14 @@
 
 package org.elasticsearch.common.joda;
 
-import org.apache.commons.lang3.StringUtils;
 import org.elasticsearch.ElasticsearchParseException;
 import org.joda.time.DateTimeZone;
 import org.joda.time.MutableDateTime;
 import org.joda.time.format.DateTimeFormatter;
 
 import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
+
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * A parser for date/time formatted text with optional date math.
@@ -38,13 +38,10 @@
 public class DateMathParser {
 
     private final FormatDateTimeFormatter dateTimeFormatter;
-    private final TimeUnit timeUnit;
 
-    public DateMathParser(FormatDateTimeFormatter dateTimeFormatter, TimeUnit timeUnit) {
-        if (dateTimeFormatter == null) throw new NullPointerException();
-        if (timeUnit == null) throw new NullPointerException();
+    public DateMathParser(FormatDateTimeFormatter dateTimeFormatter) {
+        checkNotNull(dateTimeFormatter);
         this.dateTimeFormatter = dateTimeFormatter;
-        this.timeUnit = timeUnit;
     }
 
     public long parse(String text, Callable<Long> now) {
@@ -195,17 +192,6 @@ private long parseMath(String mathString, long time, boolean roundUp, DateTimeZo
     }
 
     private long parseDateTime(String value, DateTimeZone timeZone) {
-
-        // first check for timestamp
-        if (value.length() > 4 && StringUtils.isNumeric(value)) {
-            try {
-                long time = Long.parseLong(value);
-                return timeUnit.toMillis(time);
-            } catch (NumberFormatException e) {
-                throw new ElasticsearchParseException("failed to parse date field [" + value + "] as timestamp", e);
-            }
-        }
-
         DateTimeFormatter parser = dateTimeFormatter.parser();
         if (timeZone != null) {
             parser = parser.withZone(timeZone);

src/main/java/org/elasticsearch/common/joda/Joda.java

@@ -27,6 +27,7 @@
 import org.joda.time.format.*;
 
 import java.util.Locale;
+import java.util.regex.Pattern;
 
 /**
  *
@@ -133,6 +134,10 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
             formatter = ISODateTimeFormat.yearMonth();
         } else if ("yearMonthDay".equals(input) || "year_month_day".equals(input)) {
             formatter = ISODateTimeFormat.yearMonthDay();
+        } else if ("epoch_second".equals(input)) {
+            formatter = new DateTimeFormatterBuilder().append(new EpochTimeParser(false)).toFormatter();
+        } else if ("epoch_millis".equals(input)) {
+            formatter = new DateTimeFormatterBuilder().append(new EpochTimeParser(true)).toFormatter();
         } else if (Strings.hasLength(input) && input.contains("||")) {
                 String[] formats = Strings.delimitedListToStringArray(input, "||");
                 DateTimeParser[] parsers = new DateTimeParser[formats.length];
@@ -192,4 +197,50 @@ public DateTimeField getField(Chronology chronology) {
             return new OffsetDateTimeField(new DividedDateTimeField(new OffsetDateTimeField(chronology.monthOfYear(), -1), QuarterOfYear, 3), 1);
         }
     };
+
+    public static class EpochTimeParser implements DateTimeParser {
+
+        private static final Pattern MILLI_SECOND_PRECISION_PATTERN = Pattern.compile("^\\d{1,13}$");
+        private static final Pattern SECOND_PRECISION_PATTERN = Pattern.compile("^\\d{1,10}$");
+
+        private final boolean hasMilliSecondPrecision;
+        private final Pattern pattern;
+
+        public EpochTimeParser(boolean hasMilliSecondPrecision) {
+            this.hasMilliSecondPrecision = hasMilliSecondPrecision;
+            this.pattern = hasMilliSecondPrecision ? MILLI_SECOND_PRECISION_PATTERN : SECOND_PRECISION_PATTERN;
+        }
+
+        @Override
+        public int estimateParsedLength() {
+            return hasMilliSecondPrecision ? 13 : 10;
+        }
+
+        @Override
+        public int parseInto(DateTimeParserBucket bucket, String text, int position) {
+            if (text.length() > estimateParsedLength() ||
+                // timestamps have to have UTC timezone
+                bucket.getZone() != DateTimeZone.UTC ||
+                pattern.matcher(text).matches() == false) {
+                return -1;
+            }
+
+            int factor = hasMilliSecondPrecision ? 1 : 1000;
+            try {
+                long millis = Long.valueOf(text) * factor;
+                DateTime dt = new DateTime(millis, DateTimeZone.UTC);
+                bucket.saveField(DateTimeFieldType.year(), dt.getYear());
+                bucket.saveField(DateTimeFieldType.monthOfYear(), dt.getMonthOfYear());
+                bucket.saveField(DateTimeFieldType.dayOfMonth(), dt.getDayOfMonth());
+                bucket.saveField(DateTimeFieldType.hourOfDay(), dt.getHourOfDay());
+                bucket.saveField(DateTimeFieldType.minuteOfHour(), dt.getMinuteOfHour());
+                bucket.saveField(DateTimeFieldType.secondOfMinute(), dt.getSecondOfMinute());
+                bucket.saveField(DateTimeFieldType.millisOfSecond(), dt.getMillisOfSecond());
+                bucket.setZone(DateTimeZone.UTC);
+            } catch (Exception e) {
+                return -1;
+            }
+            return text.length();
+        }
+    };
 }

src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java

@@ -46,12 +46,7 @@
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.analysis.NumericDateAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataType;
-import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.Mapper;
-import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MergeMappingException;
-import org.elasticsearch.index.mapper.MergeResult;
-import org.elasticsearch.index.mapper.ParseContext;
+import org.elasticsearch.index.mapper.*;
 import org.elasticsearch.index.mapper.core.LongFieldMapper.CustomLongNumericField;
 import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.search.internal.SearchContext;
@@ -223,7 +218,7 @@ public String toString(String s) {
 
         protected FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
         protected TimeUnit timeUnit = Defaults.TIME_UNIT;
-        protected DateMathParser dateMathParser = new DateMathParser(dateTimeFormatter, timeUnit);
+        protected DateMathParser dateMathParser = new DateMathParser(dateTimeFormatter);
 
         public DateFieldType() {}
 
@@ -245,7 +240,7 @@ public FormatDateTimeFormatter dateTimeFormatter() {
         public void setDateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
             checkIfFrozen();
             this.dateTimeFormatter = dateTimeFormatter;
-            this.dateMathParser = new DateMathParser(dateTimeFormatter, timeUnit);
+            this.dateMathParser = new DateMathParser(dateTimeFormatter);
         }
 
         public TimeUnit timeUnit() {
@@ -255,7 +250,7 @@ public TimeUnit timeUnit() {
         public void setTimeUnit(TimeUnit timeUnit) {
             checkIfFrozen();
             this.timeUnit = timeUnit;
-            this.dateMathParser = new DateMathParser(dateTimeFormatter, timeUnit);
+            this.dateMathParser = new DateMathParser(dateTimeFormatter);
         }
 
         protected DateMathParser dateMathParser() {
@@ -365,9 +360,6 @@ private Query innerRangeQuery(Object lowerTerm, Object upperTerm, boolean includ
         }
 
         public long parseToMilliseconds(Object value, boolean inclusive, @Nullable DateTimeZone zone, @Nullable DateMathParser forcedDateParser) {
-            if (value instanceof Number) {
-                return ((Number) value).longValue();
-            }
             DateMathParser dateParser = dateMathParser();
             if (forcedDateParser != null) {
                 dateParser = forcedDateParser;
@@ -434,25 +426,20 @@ protected boolean customBoost() {
     @Override
     protected void innerParseCreateField(ParseContext context, List<Field> fields) throws IOException {
         String dateAsString = null;
-        Long value = null;
         float boost = this.fieldType.boost();
         if (context.externalValueSet()) {
             Object externalValue = context.externalValue();
-            if (externalValue instanceof Number) {
-                value = ((Number) externalValue).longValue();
-            } else {
-                dateAsString = (String) externalValue;
-                if (dateAsString == null) {
-                    dateAsString = nullValue;
-                }
+            dateAsString = (String) externalValue;
+            if (dateAsString == null) {
+                dateAsString = nullValue;
             }
         } else {
             XContentParser parser = context.parser();
             XContentParser.Token token = parser.currentToken();
             if (token == XContentParser.Token.VALUE_NULL) {
                 dateAsString = nullValue;
             } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                value = parser.longValue(coerce.value());
+                dateAsString = parser.text();
             } else if (token == XContentParser.Token.START_OBJECT) {
                 String currentFieldName = null;
                 while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
@@ -462,8 +449,6 @@ protected void innerParseCreateField(ParseContext context, List<Field> fields) t
                         if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) {
                             if (token == XContentParser.Token.VALUE_NULL) {
                                 dateAsString = nullValue;
-                            } else if (token == XContentParser.Token.VALUE_NUMBER) {
-                                value = parser.longValue(coerce.value());
                             } else {
                                 dateAsString = parser.text();
                             }
@@ -479,14 +464,12 @@ protected void innerParseCreateField(ParseContext context, List<Field> fields) t
             }
         }
 
+        Long value = null;
         if (dateAsString != null) {
-            assert value == null;
             if (context.includeInAll(includeInAll, this)) {
                 context.allEntries().addText(fieldType.names().fullName(), dateAsString, boost);
             }
             value = fieldType().parseStringValue(dateAsString);
-        } else if (value != null) {
-            value = ((DateFieldType)fieldType).timeUnit().toMillis(value);
         }
 
         if (value != null) {

src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java

@@ -58,7 +58,7 @@ public class TimestampFieldMapper extends DateFieldMapper implements RootMapper
 
     public static final String NAME = "_timestamp";
     public static final String CONTENT_TYPE = "_timestamp";
-    public static final String DEFAULT_DATE_TIME_FORMAT = "dateOptionalTime";
+    public static final String DEFAULT_DATE_TIME_FORMAT = "epoch_millis||dateOptionalTime";
 
     public static class Defaults extends DateFieldMapper.Defaults {
         public static final String NAME = "_timestamp";

src/main/java/org/elasticsearch/index/query/RangeQueryParser.java

@@ -102,7 +102,7 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
                         } else if ("time_zone".equals(currentFieldName) || "timeZone".equals(currentFieldName)) {
                             timeZone = DateTimeZone.forID(parser.text());
                         } else if ("format".equals(currentFieldName)) {
-                            forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()), DateFieldMapper.Defaults.TIME_UNIT);
+                            forcedDateParser = new DateMathParser(Joda.forPattern(parser.text()));
                         } else {
                             throw new QueryParsingException(parseContext, "[range] query does not support [" + currentFieldName + "]");
                         }
@@ -123,11 +123,6 @@ public Query parse(QueryParseContext parseContext) throws IOException, QueryPars
         FieldMapper mapper = parseContext.fieldMapper(fieldName);
         if (mapper != null) {
             if (mapper instanceof DateFieldMapper) {
-                if ((from instanceof Number || to instanceof Number) && timeZone != null) {
-                    throw new QueryParsingException(parseContext,
-                            "[range] time_zone when using ms since epoch format as it's UTC based can not be applied to [" + fieldName
-                                    + "]");
-                }
                 query = ((DateFieldMapper) mapper).fieldType().rangeQuery(from, to, includeLower, includeUpper, timeZone, forcedDateParser, parseContext);
             } else  {
                 if (timeZone != null) {

src/main/java/org/elasticsearch/search/aggregations/support/format/ValueFormat.java

@@ -68,7 +68,7 @@ public static class DateTime extends Patternable<DateTime> {
         public static final DateTime DEFAULT = new DateTime(DateFieldMapper.Defaults.DATE_TIME_FORMATTER.format(), ValueFormatter.DateTime.DEFAULT, ValueParser.DateMath.DEFAULT);
 
         public static DateTime format(String format) {
-            return new DateTime(format, new ValueFormatter.DateTime(format), new ValueParser.DateMath(format, DateFieldMapper.Defaults.TIME_UNIT));
+            return new DateTime(format, new ValueFormatter.DateTime(format), new ValueParser.DateMath(format));
         }
 
         public static DateTime mapper(DateFieldMapper mapper) {

src/main/java/org/elasticsearch/search/aggregations/support/format/ValueParser.java

@@ -32,7 +32,6 @@
 import java.text.ParseException;
 import java.util.Locale;
 import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
 
 /**
  *
@@ -81,12 +80,12 @@ public double parseDouble(String value, SearchContext searchContext) {
      */
     static class DateMath implements ValueParser {
 
-        public static final DateMath DEFAULT = new ValueParser.DateMath(new DateMathParser(DateFieldMapper.Defaults.DATE_TIME_FORMATTER, DateFieldMapper.Defaults.TIME_UNIT));
+        public static final DateMath DEFAULT = new ValueParser.DateMath(new DateMathParser(DateFieldMapper.Defaults.DATE_TIME_FORMATTER));
 
         private DateMathParser parser;
 
-        public DateMath(String format, TimeUnit timeUnit) {
-            this(new DateMathParser(Joda.forPattern(format), timeUnit));
+        public DateMath(String format) {
+            this(new DateMathParser(Joda.forPattern(format)));
         }
 
         public DateMath(DateMathParser parser) {
@@ -110,7 +109,7 @@ public double parseDouble(String value, SearchContext searchContext) {
         }
 
         public static DateMath mapper(DateFieldMapper mapper) {
-            return new DateMath(new DateMathParser(mapper.fieldType().dateTimeFormatter(), DateFieldMapper.Defaults.TIME_UNIT));
+            return new DateMath(new DateMathParser(mapper.fieldType().dateTimeFormatter()));
         }
     }