Skip to content

Commit

Permalink
Generate timestamp when path is null
Browse files Browse the repository at this point in the history 
Index process fails when having `_timestamp` enabled and `path` option is set.
It fails with a `TimestampParsingException[failed to parse timestamp [null]]` message.

Reproduction:

```
DELETE test
PUT  test
{
    "mappings": {
        "test": {
            "_timestamp" : {
                "enabled" : "yes",
                "path" : "post_date"
            }
        }
    }
}
PUT test/test/1
{
  "foo": "bar"
}
```

You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "default" : null
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting timestamp format:

```
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "format" : "YYYY-MM-dd",
            "default" : "1970-01-01"
        }
    }
}
```

If you don't provide any timestamp value, indexation will fail.

Closes elastic#4718.
  • Loading branch information
@dadoonet
dadoonet committed Jul 28, 2014
1 parent 264d59c commit 472fc35
Show file tree
Hide file tree
Showing 6 changed files with 331 additions and 31 deletions.
46 changes: 45 additions & 1 deletion docs/reference/mapping/fields/timestamp-field.asciidoc
View file
Expand Up @@ -4,7 +4,7 @@
The `_timestamp` field allows to automatically index the timestamp of a
document. It can be provided externally via the index request or in the
`_source`. If it is not provided externally it will be automatically set
to the date the document was processed by the indexing chain.
to a <<mapping-timestamp-field-default,default date>>.

[float]
==== enabled
Expand Down Expand Up @@ -60,6 +60,7 @@ Note, using `path` without explicit timestamp value provided requires an
additional (though quite fast) parsing phase.

[float]
[[mapping-timestamp-field-format]]
==== format

You can define the <<mapping-date-format,date
Expand All @@ -80,3 +81,46 @@ format>> used to parse the provided timestamp value. For example:

Note, the default format is `dateOptionalTime`. The timestamp value will
first be parsed as a number and if it fails the format will be tried.

[float]
[[mapping-timestamp-field-default]]
==== default

You can define a default value for when timestamp is not provided
within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"default" : null
}
}
}
--------------------------------------------------

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:

[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"format" : "YYYY-MM-dd",
"default" : "1970-01-01"
}
}
}
--------------------------------------------------

If you don't provide any timestamp value, indexation will fail.

18 changes: 16 additions & 2 deletions src/main/java/org/elasticsearch/action/index/IndexRequest.java
View file
Expand Up @@ -23,6 +23,7 @@
import org.elasticsearch.*;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
Expand Down Expand Up @@ -574,7 +575,9 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet
}
if (parseContext.shouldParseTimestamp()) {
timestamp = parseContext.timestamp();
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
if (timestamp != null) {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(timestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
} catch (MapperParsingException e) {
throw e;
Expand Down Expand Up @@ -613,7 +616,18 @@ public void process(MetaData metaData, String aliasOrIndex, @Nullable MappingMet

// generate timestamp if not provided, we always have one post this stage...
if (timestamp == null) {
timestamp = Long.toString(System.currentTimeMillis());
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
if (mappingMd != null && mappingMd.timestamp() != null) {
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (!Strings.hasText(defaultTimestamp)) {
throw new TimestampParsingException("timestamp is required by mapping");
}
if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
timestamp = MappingMetaData.Timestamp.parseStringTimestamp(defaultTimestamp, mappingMd.timestamp().dateTimeFormatter());
}
}
}

Expand Down
24 changes: 19 additions & 5 deletions src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Expand Up @@ -175,7 +175,8 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi
}


public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT);
public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);

private final boolean enabled;

Expand All @@ -187,7 +188,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi

private final FormatDateTimeFormatter dateTimeFormatter;

public Timestamp(boolean enabled, String path, String format) {
private final String defaultTimestamp;

public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
this.enabled = enabled;
this.path = path;
if (path == null) {
Expand All @@ -197,6 +200,7 @@ public Timestamp(boolean enabled, String path, String format) {
}
this.format = format;
this.dateTimeFormatter = Joda.forPattern(format);
this.defaultTimestamp = defaultTimestamp;
}

public boolean enabled() {
Expand All @@ -219,6 +223,10 @@ public String format() {
return this.format;
}

public String defaultTimestamp() {
return this.defaultTimestamp;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand All @@ -233,6 +241,7 @@ public boolean equals(Object o) {
if (enabled != timestamp.enabled) return false;
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;

return true;
Expand All @@ -245,6 +254,7 @@ public int hashCode() {
result = 31 * result + (format != null ? format.hashCode() : 0);
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
return result;
}
}
Expand All @@ -263,7 +273,7 @@ public MappingMetaData(DocumentMapper docMapper) {
this.source = docMapper.mappingSource();
this.id = new Id(docMapper.idFieldMapper().path());
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
this.hasParentField = docMapper.parentFieldMapper().active();
}

Expand Down Expand Up @@ -328,6 +338,7 @@ private void initMappers(Map<String, Object> withoutType) {
boolean enabled = false;
String path = null;
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Expand All @@ -338,9 +349,11 @@ private void initMappers(Map<String, Object> withoutType) {
path = fieldNode.toString();
} else if (fieldName.equals("format")) {
format = fieldNode.toString();
} else if (fieldName.equals("default")) {
defaultTimestamp = fieldNode.toString();
}
}
this.timestamp = new Timestamp(enabled, path, format);
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
} else {
this.timestamp = Timestamp.EMPTY;
}
Expand Down Expand Up @@ -528,6 +541,7 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
out.writeBoolean(false);
}
out.writeString(mappingMd.timestamp().format());
out.writeString(mappingMd.timestamp().defaultTimestamp());
out.writeBoolean(mappingMd.hasParentField());
}

Expand Down Expand Up @@ -565,7 +579,7 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
// routing
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
// timestamp
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString());
Timestamp timestamp = new Timestamp(in.readBoolean(), in.readBoolean() ? in.readString() : null, in.readString(), in.readString());
final boolean hasParentField = in.readBoolean();
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
}
Expand Down
29 changes: 24 additions & 5 deletions src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Expand Up @@ -70,13 +70,15 @@ public static class Defaults extends DateFieldMapper.Defaults {
public static final EnabledAttributeMapper ENABLED = EnabledAttributeMapper.UNSET_DISABLED;
public static final String PATH = null;
public static final FormatDateTimeFormatter DATE_TIME_FORMATTER = Joda.forPattern(DEFAULT_DATE_TIME_FORMAT);
public static final String DEFAULT_TIMESTAMP = "now";
}

public static class Builder extends NumberFieldMapper.Builder<Builder, TimestampFieldMapper> {

private EnabledAttributeMapper enabledState = EnabledAttributeMapper.UNSET_DISABLED;
private String path = Defaults.PATH;
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;

public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
Expand All @@ -97,14 +99,19 @@ public Builder dateTimeFormatter(FormatDateTimeFormatter dateTimeFormatter) {
return builder;
}

public Builder defaultTimestamp(String defaultTimestamp) {
this.defaultTimestamp = defaultTimestamp;
return builder;
}

@Override
public TimestampFieldMapper build(BuilderContext context) {
boolean roundCeil = Defaults.ROUND_CEIL;
if (context.indexSettings() != null) {
Settings settings = context.indexSettings();
roundCeil = settings.getAsBoolean("index.mapping.date.round_ceil", settings.getAsBoolean("index.mapping.date.parse_upper_inclusive", Defaults.ROUND_CEIL));
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, roundCeil,
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp, roundCeil,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
Expand All @@ -124,6 +131,8 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
builder.path(fieldNode.toString());
} else if (fieldName.equals("format")) {
builder.dateTimeFormatter(parseDateTimeFormatter(builder.name(), fieldNode.toString()));
} else if (fieldName.equals("default")) {
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
}
}
return builder;
Expand All @@ -134,15 +143,16 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
private EnabledAttributeMapper enabledState;

private final String path;
private final String defaultTimestamp;

public TimestampFieldMapper() {
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER,
this(new FieldType(Defaults.FIELD_TYPE), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.ROUND_CEIL, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, ImmutableSettings.EMPTY);
}

protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, boolean roundCeil,
Explicit<Boolean> ignoreMalformed,Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp, boolean roundCeil,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
super(new Names(Defaults.NAME, Defaults.NAME, Defaults.NAME, Defaults.NAME), dateTimeFormatter,
Expand All @@ -152,6 +162,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
indexSettings, MultiFields.empty(), null);
this.enabledState = enabledState;
this.path = path;
this.defaultTimestamp = defaultTimestamp;
}

@Override
Expand All @@ -167,6 +178,10 @@ public String path() {
return this.path;
}

public String defaultTimestamp() {
return this.defaultTimestamp;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand Down Expand Up @@ -226,7 +241,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
// if all are defaults, no sense to write it at all
if (!includeDefaults && fieldType.indexed() == Defaults.FIELD_TYPE.indexed() && customFieldDataSettings == null &&
fieldType.stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED && path == Defaults.PATH
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
&& dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())
&& Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
return builder;
}
builder.startObject(CONTENT_TYPE);
Expand All @@ -246,6 +262,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeDefaults || !dateTimeFormatter.format().equals(Defaults.DATE_TIME_FORMATTER.format())) {
builder.field("format", dateTimeFormatter.format());
}
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
builder.field("default", defaultTimestamp);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {
Expand Down

0 comments on commit 472fc35

Please sign in to comment.