Skip to content

Commit

Permalink
[Mapper] Add ignore_missing option to timestamp
Browse files Browse the repository at this point in the history 
Related to #9049.

By default, the default value for `timestamp` is `now` which means the date the document was processed by the indexing chain.

You can now reject documents which not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):

```js
{
    "tweet" : {
        "_timestamp" : {
            "enabled" : true,
            "ignore_missing" : false
        }
    }
}
```

When you update the cluster to 1.5 or master, this index created with 1.4 we automatically migrate an index created with 1.4 to the 1.5 syntax.

Let say you have defined this in elasticsearch 1.4.x:

```js
DELETE test
PUT test
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 0
  }
}
PUT test/type/_mapping
{
  "type" : {
      "_timestamp" : {
          "enabled" : true,
          "default" : null
      }
  }
}
```

After migration, the mapping become:

```js
{
   "test": {
      "mappings": {
         "type": {
            "_timestamp": {
               "enabled": true,
               "store": false,
               "ignore_missing": false
            },
            "properties": {}
         }
      }
   }
}
```

Closes #8882.
  • Loading branch information
@dadoonet
dadoonet committed Jan 20, 2015
1 parent fcb0186 commit fb10346
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 146 deletions.
13 changes: 9 additions & 4 deletions docs/reference/mapping/fields/timestamp-field.asciidoc
View file
Expand Up @@ -91,22 +91,20 @@ within the index request or in the `_source` document.

By default, the default value is `now` which means the date the document was processed by the indexing chain.

You can disable that default value by setting `default` to `null`. It means that `timestamp` is mandatory:
You can reject documents which do not provide a `timestamp` value by setting `ignore_missing` to false (default to `true`):

[source,js]
--------------------------------------------------
{
"tweet" : {
"_timestamp" : {
"enabled" : true,
"default" : null
"ignore_missing" : false
}
}
}
--------------------------------------------------

If you don't provide any timestamp value, indexation will fail.

You can also set the default value to any date respecting <<mapping-timestamp-field-format,timestamp format>>:

[source,js]
Expand All @@ -124,3 +122,10 @@ You can also set the default value to any date respecting <<mapping-timestamp-fi

If you don't provide any timestamp value, _timestamp will be set to this default value.

coming[1.5.0]

In elasticsearch 1.4, we allowed setting explicitly `"default":null` which is not possible anymore
as we added a new `ignore_missing`setting.
When reading an index created with elasticsearch 1.4 and using this, we automatically update it by
removing `"default": null` and setting `"ignore_missing": false`

19 changes: 10 additions & 9 deletions src/main/java/org/elasticsearch/action/index/IndexRequest.java
View file
Expand Up @@ -20,12 +20,11 @@
package org.elasticsearch.action.index;

import com.google.common.base.Charsets;
import org.elasticsearch.*;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.RoutingMissingException;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.action.DocumentRequest;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.ElasticsearchGenerationException;
import org.elasticsearch.ElasticsearchIllegalArgumentException;
import org.elasticsearch.ElasticsearchParseException;
import org.elasticsearch.action.*;
import org.elasticsearch.action.support.replication.ShardReplicationOperationRequest;
import org.elasticsearch.client.Requests;
import org.elasticsearch.cluster.metadata.MappingMetaData;
Expand Down Expand Up @@ -659,11 +658,13 @@ public void process(MetaData metaData, @Nullable MappingMetaData mappingMd, bool
if (timestamp == null) {
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
if (mappingMd != null && mappingMd.timestamp() != null) {
// If we explicitly ask to reject null timestamp
if (mappingMd.timestamp().ignoreMissing() != null && mappingMd.timestamp().ignoreMissing() == false) {
throw new TimestampParsingException("timestamp is required by mapping");
}
defaultTimestamp = mappingMd.timestamp().defaultTimestamp();
}
if (!Strings.hasText(defaultTimestamp)) {
throw new TimestampParsingException("timestamp is required by mapping");
}

if (defaultTimestamp.equals(TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP)) {
timestamp = Long.toString(System.currentTimeMillis());
} else {
Expand Down
40 changes: 35 additions & 5 deletions src/main/java/org/elasticsearch/cluster/metadata/MappingMetaData.java
View file
Expand Up @@ -177,7 +177,7 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi


public static final Timestamp EMPTY = new Timestamp(false, null, TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT,
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP);
TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP, null);

private final boolean enabled;

Expand All @@ -191,7 +191,9 @@ public static String parseStringTimestamp(String timestampAsString, FormatDateTi

private final String defaultTimestamp;

public Timestamp(boolean enabled, String path, String format, String defaultTimestamp) {
private final Boolean ignoreMissing;

public Timestamp(boolean enabled, String path, String format, String defaultTimestamp, Boolean ignoreMissing) {
this.enabled = enabled;
this.path = path;
if (path == null) {
Expand All @@ -202,6 +204,7 @@ public Timestamp(boolean enabled, String path, String format, String defaultTime
this.format = format;
this.dateTimeFormatter = Joda.forPattern(format);
this.defaultTimestamp = defaultTimestamp;
this.ignoreMissing = ignoreMissing;
}

public boolean enabled() {
Expand Down Expand Up @@ -232,6 +235,10 @@ public boolean hasDefaultTimestamp() {
return this.defaultTimestamp != null;
}

public Boolean ignoreMissing() {
return ignoreMissing;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand All @@ -247,6 +254,7 @@ public boolean equals(Object o) {
if (format != null ? !format.equals(timestamp.format) : timestamp.format != null) return false;
if (path != null ? !path.equals(timestamp.path) : timestamp.path != null) return false;
if (defaultTimestamp != null ? !defaultTimestamp.equals(timestamp.defaultTimestamp) : timestamp.defaultTimestamp != null) return false;
if (ignoreMissing != null ? !ignoreMissing.equals(timestamp.ignoreMissing) : timestamp.ignoreMissing != null) return false;
if (!Arrays.equals(pathElements, timestamp.pathElements)) return false;

return true;
Expand All @@ -260,6 +268,7 @@ public int hashCode() {
result = 31 * result + (pathElements != null ? Arrays.hashCode(pathElements) : 0);
result = 31 * result + (dateTimeFormatter != null ? dateTimeFormatter.hashCode() : 0);
result = 31 * result + (defaultTimestamp != null ? defaultTimestamp.hashCode() : 0);
result = 31 * result + (ignoreMissing != null ? ignoreMissing.hashCode() : 0);
return result;
}
}
Expand All @@ -278,7 +287,9 @@ public MappingMetaData(DocumentMapper docMapper) {
this.source = docMapper.mappingSource();
this.id = new Id(docMapper.idFieldMapper().path());
this.routing = new Routing(docMapper.routingFieldMapper().required(), docMapper.routingFieldMapper().path());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(), docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp());
this.timestamp = new Timestamp(docMapper.timestampFieldMapper().enabled(), docMapper.timestampFieldMapper().path(),
docMapper.timestampFieldMapper().dateTimeFormatter().format(), docMapper.timestampFieldMapper().defaultTimestamp(),
docMapper.timestampFieldMapper().ignoreMissing());
this.hasParentField = docMapper.parentFieldMapper().active();
}

Expand Down Expand Up @@ -344,6 +355,7 @@ private void initMappers(Map<String, Object> withoutType) {
String path = null;
String format = TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT;
String defaultTimestamp = TimestampFieldMapper.Defaults.DEFAULT_TIMESTAMP;
Boolean ignoreMissing = null;
Map<String, Object> timestampNode = (Map<String, Object>) withoutType.get("_timestamp");
for (Map.Entry<String, Object> entry : timestampNode.entrySet()) {
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Expand All @@ -356,9 +368,11 @@ private void initMappers(Map<String, Object> withoutType) {
format = fieldNode.toString();
} else if (fieldName.equals("default") && fieldNode != null) {
defaultTimestamp = fieldNode.toString();
} else if (fieldName.equals("ignore_missing")) {
ignoreMissing = nodeBooleanValue(fieldNode);
}
}
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp);
this.timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
} else {
this.timestamp = Timestamp.EMPTY;
}
Expand Down Expand Up @@ -542,6 +556,10 @@ public static void writeTo(MappingMetaData mappingMd, StreamOutput out) throws I
out.writeOptionalString(mappingMd.timestamp().path());
out.writeString(mappingMd.timestamp().format());
out.writeOptionalString(mappingMd.timestamp().defaultTimestamp());
// TODO Remove the test in elasticsearch 2.0.0
if (out.getVersion().onOrAfter(Version.V_1_5_0)) {
out.writeOptionalBoolean(mappingMd.timestamp().ignoreMissing());
}
out.writeBoolean(mappingMd.hasParentField());
}

Expand Down Expand Up @@ -579,7 +597,19 @@ public static MappingMetaData readFrom(StreamInput in) throws IOException {
// routing
Routing routing = new Routing(in.readBoolean(), in.readBoolean() ? in.readString() : null);
// timestamp
final Timestamp timestamp = new Timestamp(in.readBoolean(), in.readOptionalString(), in.readString(), in.readOptionalString());

boolean enabled = in.readBoolean();
String path = in.readOptionalString();
String format = in.readString();
String defaultTimestamp = in.readOptionalString();
Boolean ignoreMissing = null;

// TODO Remove the test in elasticsearch 2.0.0
if (in.getVersion().onOrAfter(Version.V_1_5_0)) {
ignoreMissing = in.readOptionalBoolean();
}

final Timestamp timestamp = new Timestamp(enabled, path, format, defaultTimestamp, ignoreMissing);
final boolean hasParentField = in.readBoolean();
return new MappingMetaData(type, source, id, routing, timestamp, hasParentField);
}
Expand Down
47 changes: 45 additions & 2 deletions src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Expand Up @@ -24,6 +24,7 @@
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.index.IndexOptions;
import org.elasticsearch.Version;
import org.elasticsearch.action.TimestampParsingException;
import org.elasticsearch.common.Explicit;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.Strings;
Expand Down Expand Up @@ -86,6 +87,7 @@ public static class Builder extends NumberFieldMapper.Builder<Builder, Timestamp
private FormatDateTimeFormatter dateTimeFormatter = Defaults.DATE_TIME_FORMATTER;
private String defaultTimestamp = Defaults.DEFAULT_TIMESTAMP;
private boolean explicitStore = false;
private Boolean ignoreMissing = null;

public Builder() {
super(Defaults.NAME, new FieldType(Defaults.FIELD_TYPE), Defaults.PRECISION_STEP_64_BIT);
Expand All @@ -111,6 +113,11 @@ public Builder defaultTimestamp(String defaultTimestamp) {
return builder;
}

public Builder ignoreMissing(boolean ignoreMissing) {
this.ignoreMissing = ignoreMissing;
return builder;
}

@Override
public Builder store(boolean store) {
explicitStore = true;
Expand All @@ -124,6 +131,7 @@ public TimestampFieldMapper build(BuilderContext context) {
fieldType.setStored(false);
}
return new TimestampFieldMapper(fieldType, docValues, enabledState, path, dateTimeFormatter, defaultTimestamp,
ignoreMissing,
ignoreMalformed(context), coerce(context), postingsProvider, docValuesProvider, normsLoading, fieldDataSettings, context.indexSettings());
}
}
Expand All @@ -133,6 +141,8 @@ public static class TypeParser implements Mapper.TypeParser {
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
TimestampFieldMapper.Builder builder = timestamp();
parseField(builder, builder.name, node, parserContext);
boolean defaultSet = false;
Boolean ignoreMissing = null;
for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
Map.Entry<String, Object> entry = iterator.next();
String fieldName = Strings.toUnderscoreCase(entry.getKey());
Expand All @@ -148,10 +158,33 @@ public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext
builder.dateTimeFormatter(parseDateTimeFormatter(fieldNode.toString()));
iterator.remove();
} else if (fieldName.equals("default")) {
builder.defaultTimestamp(fieldNode == null ? null : fieldNode.toString());
if (fieldNode == null) {
if (parserContext.indexVersionCreated().onOrAfter(Version.V_1_4_0_Beta1) &&
parserContext.indexVersionCreated().before(Version.V_1_5_0)) {
// We are reading an index created in 1.4 with feature #7036
// `default: null` was explicitly set. We need to change this index to
// `ignore_missing: false`
builder.ignoreMissing(false);
} else {
throw new TimestampParsingException("default timestamp can not be set to null");
}
} else {
builder.defaultTimestamp(fieldNode.toString());
defaultSet = true;
}
iterator.remove();
} else if (fieldName.equals("ignore_missing")) {
ignoreMissing = nodeBooleanValue(fieldNode);
builder.ignoreMissing(ignoreMissing);
iterator.remove();
}
}

// We can not accept a default value and rejecting null values at the same time
if (defaultSet && (ignoreMissing != null && ignoreMissing == false)) {
throw new TimestampParsingException("default timestamp can not be set with ignore_missing set to false");
}

return builder;
}
}
Expand All @@ -165,14 +198,16 @@ private static FieldType defaultFieldType(Settings settings) {
private final String path;
private final String defaultTimestamp;
private final FieldType defaultFieldType;
private final Boolean ignoreMissing;

public TimestampFieldMapper(Settings indexSettings) {
this(new FieldType(defaultFieldType(indexSettings)), null, Defaults.ENABLED, Defaults.PATH, Defaults.DATE_TIME_FORMATTER, Defaults.DEFAULT_TIMESTAMP,
Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
null, Defaults.IGNORE_MALFORMED, Defaults.COERCE, null, null, null, null, indexSettings);
}

protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAttributeMapper enabledState, String path,
FormatDateTimeFormatter dateTimeFormatter, String defaultTimestamp,
Boolean ignoreMissing,
Explicit<Boolean> ignoreMalformed, Explicit<Boolean> coerce, PostingsFormatProvider postingsProvider,
DocValuesFormatProvider docValuesProvider, Loading normsLoading,
@Nullable Settings fieldDataSettings, Settings indexSettings) {
Expand All @@ -185,6 +220,7 @@ protected TimestampFieldMapper(FieldType fieldType, Boolean docValues, EnabledAt
this.path = path;
this.defaultTimestamp = defaultTimestamp;
this.defaultFieldType = defaultFieldType(indexSettings);
this.ignoreMissing = ignoreMissing;
}

@Override
Expand All @@ -204,6 +240,10 @@ public String defaultTimestamp() {
return this.defaultTimestamp;
}

public Boolean ignoreMissing() {
return this.ignoreMissing;
}

public FormatDateTimeFormatter dateTimeFormatter() {
return this.dateTimeFormatter;
}
Expand Down Expand Up @@ -292,6 +332,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
if (includeDefaults || !Defaults.DEFAULT_TIMESTAMP.equals(defaultTimestamp)) {
builder.field("default", defaultTimestamp);
}
if (includeDefaults || ignoreMissing != null) {
builder.field("ignore_missing", ignoreMissing);
}
if (customFieldDataSettings != null) {
builder.field("fielddata", (Map) customFieldDataSettings.getAsMap());
} else if (includeDefaults) {
Expand Down

0 comments on commit fb10346

Please sign in to comment.