Skip to content

Commit

Permalink
Dates: More strict parsing of ISO dates
Browse files Browse the repository at this point in the history 
If you are using the default date or the named identifiers of dates,
the current implementation was allowed to read a year with only one
digit. In order to make this more strict, this fixes a year to be at
least 4 digits. Same applies for month, day, hour, minute, seconds.

Also the new default is `strictDateOptionalTime` for indices created
with Elasticsearch 2.0 or newer.

In addition a couple of not exposed date formats have been exposed, as they
have been mentioned in the documentation.

Closes #6158
  • Loading branch information
@spinscale
spinscale committed Jul 7, 2015
1 parent 35ddc74 commit b612cab
Show file tree
Hide file tree
Showing 16 changed files with 2,709 additions and 68 deletions.
111 changes: 110 additions & 1 deletion core/src/main/java/org/elasticsearch/common/joda/Joda.java
View file
Expand Up @@ -118,6 +118,8 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
formatter = ISODateTimeFormat.ordinalDateTimeNoMillis();
} else if ("time".equals(input)) {
formatter = ISODateTimeFormat.time();
} else if ("timeNoMillis".equals(input) || "time_no_millis".equals(input)) {
formatter = ISODateTimeFormat.timeNoMillis();
} else if ("tTime".equals(input) || "t_time".equals(input)) {
formatter = ISODateTimeFormat.tTime();
} else if ("tTimeNoMillis".equals(input) || "t_time_no_millis".equals(input)) {
Expand All @@ -126,10 +128,14 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
formatter = ISODateTimeFormat.weekDate();
} else if ("weekDateTime".equals(input) || "week_date_time".equals(input)) {
formatter = ISODateTimeFormat.weekDateTime();
} else if ("weekDateTimeNoMillis".equals(input) || "week_date_time_no_millis".equals(input)) {
formatter = ISODateTimeFormat.weekDateTimeNoMillis();
} else if ("weekyear".equals(input) || "week_year".equals(input)) {
formatter = ISODateTimeFormat.weekyear();
} else if ("weekyearWeek".equals(input)) {
} else if ("weekyearWeek".equals(input) || "weekyear_week".equals(input)) {
formatter = ISODateTimeFormat.weekyearWeek();
} else if ("weekyearWeekDay".equals(input) || "weekyear_week_day".equals(input)) {
formatter = ISODateTimeFormat.weekyearWeekDay();
} else if ("year".equals(input)) {
formatter = ISODateTimeFormat.year();
} else if ("yearMonth".equals(input) || "year_month".equals(input)) {
Expand All @@ -140,6 +146,77 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
formatter = new DateTimeFormatterBuilder().append(new EpochTimePrinter(false), new EpochTimeParser(false)).toFormatter();
} else if ("epoch_millis".equals(input)) {
formatter = new DateTimeFormatterBuilder().append(new EpochTimePrinter(true), new EpochTimeParser(true)).toFormatter();
// strict date formats here, must be at least 4 digits for year and two for months and two for day
} else if ("strictBasicWeekDate".equals(input) || "strict_basic_week_date".equals(input)) {
formatter = StrictISODateTimeFormat.basicWeekDate();
} else if ("strictBasicWeekDateTime".equals(input) || "strict_basic_week_date_time".equals(input)) {
formatter = StrictISODateTimeFormat.basicWeekDateTime();
} else if ("strictBasicWeekDateTimeNoMillis".equals(input) || "strict_basic_week_date_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.basicWeekDateTimeNoMillis();
} else if ("strictDate".equals(input) || "strict_date".equals(input)) {
formatter = StrictISODateTimeFormat.date();
} else if ("strictDateHour".equals(input) || "strict_date_hour".equals(input)) {
formatter = StrictISODateTimeFormat.dateHour();
} else if ("strictDateHourMinute".equals(input) || "strict_date_hour_minute".equals(input)) {
formatter = StrictISODateTimeFormat.dateHourMinute();
} else if ("strictDateHourMinuteSecond".equals(input) || "strict_date_hour_minute_second".equals(input)) {
formatter = StrictISODateTimeFormat.dateHourMinuteSecond();
} else if ("strictDateHourMinuteSecondFraction".equals(input) || "strict_date_hour_minute_second_fraction".equals(input)) {
formatter = StrictISODateTimeFormat.dateHourMinuteSecondFraction();
} else if ("strictDateHourMinuteSecondMillis".equals(input) || "strict_date_hour_minute_second_millis".equals(input)) {
formatter = StrictISODateTimeFormat.dateHourMinuteSecondMillis();
} else if ("strictDateOptionalTime".equals(input) || "strict_date_optional_time".equals(input)) {
// in this case, we have a separate parser and printer since the dataOptionalTimeParser can't print
// this sucks we should use the root local by default and not be dependent on the node
return new FormatDateTimeFormatter(input,
StrictISODateTimeFormat.dateOptionalTimeParser().withZone(DateTimeZone.UTC),
StrictISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC), locale);
} else if ("strictDateTime".equals(input) || "strict_date_time".equals(input)) {
formatter = StrictISODateTimeFormat.dateTime();
} else if ("strictDateTimeNoMillis".equals(input) || "strict_date_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.dateTimeNoMillis();
} else if ("strictHour".equals(input) || "strict_hour".equals(input)) {
formatter = StrictISODateTimeFormat.hour();
} else if ("strictHourMinute".equals(input) || "strict_hour_minute".equals(input)) {
formatter = StrictISODateTimeFormat.hourMinute();
} else if ("strictHourMinuteSecond".equals(input) || "strict_hour_minute_second".equals(input)) {
formatter = StrictISODateTimeFormat.hourMinuteSecond();
} else if ("strictHourMinuteSecondFraction".equals(input) || "strict_hour_minute_second_fraction".equals(input)) {
formatter = StrictISODateTimeFormat.hourMinuteSecondFraction();
} else if ("strictHourMinuteSecondMillis".equals(input) || "strict_hour_minute_second_millis".equals(input)) {
formatter = StrictISODateTimeFormat.hourMinuteSecondMillis();
} else if ("strictOrdinalDate".equals(input) || "strict_ordinal_date".equals(input)) {
formatter = StrictISODateTimeFormat.ordinalDate();
} else if ("strictOrdinalDateTime".equals(input) || "strict_ordinal_date_time".equals(input)) {
formatter = StrictISODateTimeFormat.ordinalDateTime();
} else if ("strictOrdinalDateTimeNoMillis".equals(input) || "strict_ordinal_date_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.ordinalDateTimeNoMillis();
} else if ("strictTime".equals(input) || "strict_time".equals(input)) {
formatter = StrictISODateTimeFormat.time();
} else if ("strictTimeNoMillis".equals(input) || "strict_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.timeNoMillis();
} else if ("strictTTime".equals(input) || "strict_t_time".equals(input)) {
formatter = StrictISODateTimeFormat.tTime();
} else if ("strictTTimeNoMillis".equals(input) || "strict_t_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.tTimeNoMillis();
} else if ("strictWeekDate".equals(input) || "strict_week_date".equals(input)) {
formatter = StrictISODateTimeFormat.weekDate();
} else if ("strictWeekDateTime".equals(input) || "strict_week_date_time".equals(input)) {
formatter = StrictISODateTimeFormat.weekDateTime();
} else if ("strictWeekDateTimeNoMillis".equals(input) || "strict_week_date_time_no_millis".equals(input)) {
formatter = StrictISODateTimeFormat.weekDateTimeNoMillis();
} else if ("strictWeekyear".equals(input) || "strict_weekyear".equals(input)) {
formatter = StrictISODateTimeFormat.weekyear();
} else if ("strictWeekyearWeek".equals(input) || "strict_weekyear_week".equals(input)) {
formatter = StrictISODateTimeFormat.weekyearWeek();
} else if ("strictWeekyearWeekDay".equals(input) || "strict_weekyear_week_day".equals(input)) {
formatter = StrictISODateTimeFormat.weekyearWeekDay();
} else if ("strictYear".equals(input) || "strict_year".equals(input)) {
formatter = StrictISODateTimeFormat.year();
} else if ("strictYearMonth".equals(input) || "strict_year_month".equals(input)) {
formatter = StrictISODateTimeFormat.yearMonth();
} else if ("strictYearMonthDay".equals(input) || "strict_year_month_day".equals(input)) {
formatter = StrictISODateTimeFormat.yearMonthDay();
} else if (Strings.hasLength(input) && input.contains("||")) {
String[] formats = Strings.delimitedListToStringArray(input, "||");
DateTimeParser[] parsers = new DateTimeParser[formats.length];
Expand Down Expand Up @@ -171,6 +248,38 @@ public static FormatDateTimeFormatter forPattern(String input, Locale locale) {
return new FormatDateTimeFormatter(input, formatter.withZone(DateTimeZone.UTC), locale);
}

public static FormatDateTimeFormatter getStrictStandardDateFormatter() {
// 2014/10/10
DateTimeFormatter shortFormatter = new DateTimeFormatterBuilder()
.appendFixedDecimal(DateTimeFieldType.year(), 4)
.appendLiteral('/')
.appendFixedDecimal(DateTimeFieldType.monthOfYear(), 2)
.appendLiteral('/')
.appendFixedDecimal(DateTimeFieldType.dayOfMonth(), 2)
.toFormatter()
.withZoneUTC();

// 2014/10/10 12:12:12
DateTimeFormatter longFormatter = new DateTimeFormatterBuilder()
.appendFixedDecimal(DateTimeFieldType.year(), 4)
.appendLiteral('/')
.appendFixedDecimal(DateTimeFieldType.monthOfYear(), 2)
.appendLiteral('/')
.appendFixedDecimal(DateTimeFieldType.dayOfMonth(), 2)
.appendLiteral(' ')
.appendFixedSignedDecimal(DateTimeFieldType.hourOfDay(), 2)
.appendLiteral(':')
.appendFixedSignedDecimal(DateTimeFieldType.minuteOfHour(), 2)
.appendLiteral(':')
.appendFixedSignedDecimal(DateTimeFieldType.secondOfMinute(), 2)
.toFormatter()
.withZoneUTC();

DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder().append(longFormatter.withZone(DateTimeZone.UTC).getPrinter(), new DateTimeParser[] {longFormatter.getParser(), shortFormatter.getParser()});

return new FormatDateTimeFormatter("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd", builder.toFormatter().withZone(DateTimeZone.UTC), Locale.ROOT);
}


public static final DurationFieldType Quarters = new DurationFieldType("quarters") {
private static final long serialVersionUID = -8167713675442491871L;
Expand Down
26 changes: 17 additions & 9 deletions core/src/main/java/org/elasticsearch/index/mapper/core/DateFieldMapper.java
View file
Expand Up @@ -69,7 +69,8 @@ public class DateFieldMapper extends NumberFieldMapper {
public static final String CONTENT_TYPE = "date";

public static class Defaults extends NumberFieldMapper.Defaults {
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("dateOptionalTime||epoch_millis", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern("strictDateOptionalTime||epoch_millis", Locale.ROOT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("dateOptionalTime", Locale.ROOT);
public static final TimeUnit TIME_UNIT = TimeUnit.MILLISECONDS;
public static final DateFieldType FIELD_TYPE = new DateFieldType();

Expand Down Expand Up @@ -123,15 +124,13 @@ public DateFieldMapper build(BuilderContext context) {
}

protected void setupFieldType(BuilderContext context) {
FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
// TODO MOVE ME OUTSIDE OF THIS SPACE?
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0)) {
boolean includesEpochFormatter = dateTimeFormatter.format().contains("epoch_");
if (!includesEpochFormatter) {
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + dateTimeFormatter.format()));
}
if (Version.indexCreated(context.indexSettings()).before(Version.V_2_0_0) &&
!fieldType().dateTimeFormatter().format().contains("epoch_")) {
String format = fieldType().timeUnit().equals(TimeUnit.SECONDS) ? "epoch_second" : "epoch_millis";
fieldType().setDateTimeFormatter(Joda.forPattern(format + "||" + fieldType().dateTimeFormatter().format()));
}

FormatDateTimeFormatter dateTimeFormatter = fieldType().dateTimeFormatter;
if (!locale.equals(dateTimeFormatter.locale())) {
fieldType().setDateTimeFormatter(new FormatDateTimeFormatter(dateTimeFormatter.format(), dateTimeFormatter.parser(), dateTimeFormatter.printer(), locale));
}
Expand Down Expand Up @@ -159,6 +158,7 @@ public static class TypeParser implements Mapper.TypeParser {
public Mapper.Builder<?, ?> parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
DateFieldMapper.Builder builder = dateField(name);
parseNumberField(builder, name, node, parserContext);
boolean configuredFormat = false;
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String propName = Strings.toUnderscoreCase(entry.getKey());
Expand All @@ -171,6 +171,7 @@ public static class TypeParser implements Mapper.TypeParser {
iterator.remove();
} else if (propName.equals("format")) {
builder.dateTimeFormatter(parseDateTimeFormatter(propNode));
configuredFormat = true;
iterator.remove();
} else if (propName.equals("numeric_resolution")) {
builder.timeUnit(TimeUnit.valueOf(propNode.toString().toUpperCase(Locale.ROOT)));
Expand All @@ -180,6 +181,13 @@ public static class TypeParser implements Mapper.TypeParser {
iterator.remove();
}
}
if (!configuredFormat) {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_2_0_0)) {
builder.dateTimeFormatter(Defaults.DATE_TIME_FORMATTER);
} else {
builder.dateTimeFormatter(Defaults.DATE_TIME_FORMATTER_BEFORE_2_0);
}
}
return builder;
}
}
Expand Down
35 changes: 26 additions & 9 deletions core/src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Expand Up @@ -24,14 +24,12 @@
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.joda.FormatDateTimeFormatter;
import org.elasticsearch.common.joda.Joda;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.analysis.NumericDateAnalyzer;
import org.elasticsearch.index.fielddata.FieldDataType;
import org.elasticsearch.index.mapper.MappedFieldType;
Expand All @@ -41,10 +39,8 @@
import org.elasticsearch.index.mapper.MergeResult;
import org.elasticsearch.index.mapper.ParseContext;
import org.elasticsearch.index.mapper.MetadataFieldMapper;
import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
import org.elasticsearch.index.mapper.core.DateFieldMapper;
import org.elasticsearch.index.mapper.core.LongFieldMapper;
import org.elasticsearch.index.mapper.core.NumberFieldMapper;

import java.io.IOException;
import java.util.Iterator;
Expand All @@ -59,15 +55,16 @@ public class TimestampFieldMapper extends MetadataFieldMapper {

public static final String NAME = "_timestamp";
public static final String CONTENT_TYPE = "_timestamp";
public static final String DEFAULT_DATE_TIME_FORMAT = "epoch_millis||dateOptionalTime";
public static final String DEFAULT_DATE_TIME_FORMAT = "epoch_millis||strictDateOptionalTime";

public static class Defaults extends DateFieldMapper.Defaults {
public static final String NAME = "_timestamp";

// TODO: this should be removed
public static final MappedFieldType PRE_20_FIELD_TYPE;
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
public static final TimestampFieldType PRE_20_FIELD_TYPE;
public static final TimestampFieldType FIELD_TYPE = new TimestampFieldType();
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER_BEFORE_2_0 = Joda.forPattern("epoch_millis||dateOptionalTime");

static {
FIELD_TYPE.setStored(true);
Expand All @@ -82,6 +79,9 @@ public static class Defaults extends DateFieldMapper.Defaults {
PRE_20_FIELD_TYPE = FIELD_TYPE.clone();
PRE_20_FIELD_TYPE.setStored(false);
PRE_20_FIELD_TYPE.setHasDocValues(false);
PRE_20_FIELD_TYPE.setDateTimeFormatter(DATE_TIME_FORMATTER_BEFORE_2_0);
PRE_20_FIELD_TYPE.setIndexAnalyzer(NumericDateAnalyzer.buildNamedAnalyzer(DATE_TIME_FORMATTER_BEFORE_2_0, Defaults.PRECISION_STEP_64_BIT));
PRE_20_FIELD_TYPE.setSearchAnalyzer(NumericDateAnalyzer.buildNamedAnalyzer(DATE_TIME_FORMATTER_BEFORE_2_0, Integer.MAX_VALUE));
PRE_20_FIELD_TYPE.freeze();
}

Expand Down Expand Up @@ -146,8 +146,23 @@ public TimestampFieldMapper build(BuilderContext context) {
if (explicitStore == false && context.indexCreatedVersion().before(Version.V_2_0_0)) {
fieldType.setStored(false);
}

if (fieldType().dateTimeFormatter().equals(Defaults.DATE_TIME_FORMATTER)) {
fieldType().setDateTimeFormatter(getDateTimeFormatter(context.indexSettings()));
}

setupFieldType(context);
return new TimestampFieldMapper(fieldType, defaultFieldType, enabledState, path, defaultTimestamp, ignoreMissing, context.indexSettings());
return new TimestampFieldMapper(fieldType, defaultFieldType, enabledState, path, defaultTimestamp,
ignoreMissing, context.indexSettings());
}
}

private static FormatDateTimeFormatter getDateTimeFormatter(Settings indexSettings) {
Version indexCreated = Version.indexCreated(indexSettings);
if (indexCreated.onOrAfter(Version.V_2_0_0)) {
return Defaults.DATE_TIME_FORMATTER;
} else {
return Defaults.DATE_TIME_FORMATTER_BEFORE_2_0;
}
}

Expand Down Expand Up @@ -341,7 +356,9 @@ && fieldType().dateTimeFormatter().format().equals(Defaults.DATE_TIME_FORMATTER.
if (indexCreatedBefore2x && (includeDefaults || path != Defaults.PATH)) {
builder.field("path", path);
}
if (includeDefaults || !fieldType().dateTimeFormatter().format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
// different format handling depending on index version
String defaultDateFormat = indexCreatedBefore2x ? Defaults.DATE_TIME_FORMATTER_BEFORE_2_0.format() : Defaults.DATE_TIME_FORMATTER.format();
if (includeDefaults || !fieldType().dateTimeFormatter().format().equals(defaultDateFormat)) {
builder.field("format", fieldType().dateTimeFormatter().format());
}
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/org/elasticsearch/index/mapper/object/RootObjectMapper.java
View file
Expand Up @@ -49,7 +49,7 @@ public static class Defaults {
public static final FormatDateTimeFormatter[] DYNAMIC_DATE_TIME_FORMATTERS =
new FormatDateTimeFormatter[]{
DateFieldMapper.Defaults.DATE_TIME_FORMATTER,
Joda.forPattern("yyyy/MM/dd HH:mm:ss||yyyy/MM/dd")
Joda.getStrictStandardDateFormatter()
};
public static final boolean DATE_DETECTION = true;
public static final boolean NUMERIC_DETECTION = false;
Expand Down

0 comments on commit b612cab

Please sign in to comment.