Skip to content

Commit

Permalink
_timestamp: enable mapper properties merging
Browse files Browse the repository at this point in the history 
Updates on the _timestamp field were silently ignored.
Now _timestamp undergoes the same merge as regular
fields. This includes exceptions if a prroperty cannot
be changed.
"path" and "default" cannot be changed.

closes elastic#5772
closes elastic#6958
partially fixes elastic#777
  • Loading branch information
@brwe
brwe committed Sep 5, 2014
1 parent 8b8cc80 commit 19e3661
Show file tree
Hide file tree
Showing 4 changed files with 235 additions and 7 deletions.
12 changes: 12 additions & 0 deletions src/main/java/org/elasticsearch/index/mapper/internal/TimestampFieldMapper.java
View file
Expand Up @@ -277,10 +277,22 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
@Override
public void merge(Mapper mergeWith, MergeContext mergeContext) throws MergeMappingException {
TimestampFieldMapper timestampFieldMapperMergeWith = (TimestampFieldMapper) mergeWith;
super.merge(mergeWith, mergeContext);
if (!mergeContext.mergeFlags().simulate()) {
if (timestampFieldMapperMergeWith.enabledState != enabledState && !timestampFieldMapperMergeWith.enabledState.unset()) {
this.enabledState = timestampFieldMapperMergeWith.enabledState;
}
} else {
if (!timestampFieldMapperMergeWith.defaultTimestamp().equals(defaultTimestamp)) {
mergeContext.addConflict("Cannot update default in _timestamp value. Value is " + defaultTimestamp.toString() + " now encountering " + timestampFieldMapperMergeWith.defaultTimestamp());
}
if (this.path != null) {
if (!path.equals(timestampFieldMapperMergeWith.path())) {
mergeContext.addConflict("Cannot update path in _timestamp value. Value is " + path + " now encountering " + timestampFieldMapperMergeWith.path());
}
} else if (timestampFieldMapperMergeWith.path() != null) {
mergeContext.addConflict("Cannot update path in _timestamp value. Value is " + path + " now encountering null");
}
}
}
}
165 changes: 159 additions & 6 deletions src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View file
Expand Up @@ -32,16 +32,14 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.DocumentMapperParser;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.*;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
import org.elasticsearch.test.ElasticsearchSingleNodeTest;
import org.junit.Test;

import java.io.IOException;
import java.util.Locale;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.*;
Expand Down Expand Up @@ -89,7 +87,7 @@ public void testDefaultValues() throws Exception {
assertThat(docMapper.timestampFieldMapper().enabled(), equalTo(TimestampFieldMapper.Defaults.ENABLED.enabled));
assertThat(docMapper.timestampFieldMapper().fieldType().stored(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.stored()));
assertThat(docMapper.timestampFieldMapper().fieldType().indexed(), equalTo(TimestampFieldMapper.Defaults.FIELD_TYPE.indexed()));
assertThat(docMapper.timestampFieldMapper().path(), equalTo(null));
assertThat(docMapper.timestampFieldMapper().path(), equalTo(TimestampFieldMapper.Defaults.PATH));
assertThat(docMapper.timestampFieldMapper().dateTimeFormatter().format(), equalTo(TimestampFieldMapper.DEFAULT_DATE_TIME_FORMAT));
}

Expand Down Expand Up @@ -390,4 +388,159 @@ public void testDefaultTimestampStream() throws IOException {
assertThat(metaData, is(expected));
}
}

@Test
public void testMergingFielddataLoadingWorks() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject()
.endObject().endObject().string();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();

DocumentMapper docMapper = parser.parse(mapping);
assertThat(docMapper.timestampFieldMapper().fieldDataType().getLoading(), equalTo(FieldMapper.Loading.LAZY));
assertThat(docMapper.timestampFieldMapper().fieldDataType().getFormat(docMapper.timestampFieldMapper().fieldDataType().getSettings()), equalTo("doc_values"));
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "yes").endObject()
.endObject().endObject().string();

DocumentMapper.MergeResult mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(false));
assertThat(mergeResult.conflicts().length, equalTo(0));
assertThat(docMapper.timestampFieldMapper().fieldDataType().getLoading(), equalTo(FieldMapper.Loading.EAGER));
assertThat(docMapper.timestampFieldMapper().fieldDataType().getFormat(docMapper.timestampFieldMapper().fieldDataType().getSettings()), equalTo("array"));
}

@Test
public void testParsingTwiceDoesNotChangeMapping() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true)
.field("index", "no")
.field("store", true)
.field("path", "foo")
.field("default", "1970-01-01")
.startObject("fielddata").field("format", "doc_values").endObject()
.endObject()
.startObject("properties")
.endObject()
.endObject().endObject().string();
logger.info(mapping);
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();

DocumentMapper docMapper = parser.parse(mapping);
docMapper.refreshSource();
docMapper = parser.parse(docMapper.mappingSource().string());
assertThat(docMapper.mappingSource().string(), equalTo(mapping));
}

@Test
public void testMergingConflicts() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true)
.startObject("fielddata").field("format", "doc_values").endObject()
.field("store", "yes")
.field("index", "analyzed")
.field("path", "foo")
.field("default", "1970-01-01")
.endObject()
.endObject().endObject().string();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();

DocumentMapper docMapper = parser.parse(mapping);
docMapper.refreshSource();
docMapper = parser.parse(docMapper.mappingSource().string());
assertThat(docMapper.timestampFieldMapper().fieldDataType().getLoading(), equalTo(FieldMapper.Loading.LAZY));
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", false)
.startObject("fielddata").field("format", "array").endObject()
.field("store", "no")
.field("index", "no")
.field("path", "bar")
.field("default", "1970-01-02")
.endObject()
.endObject().endObject().string();

DocumentMapper.MergeResult mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
String[] expectedConflicts = {"mapper [_timestamp] has different index values", "mapper [_timestamp] has different store values", "Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02", "Cannot update path in _timestamp value. Value is foo now encountering bar"};
assertThat(mergeResult.conflicts().length, equalTo(expectedConflicts.length));
for (String conflict : expectedConflicts) {
assertThat(conflict, isIn(mergeResult.conflicts()));
}
assertThat(docMapper.timestampFieldMapper().fieldDataType().getLoading(), equalTo(FieldMapper.Loading.LAZY));
assertTrue(docMapper.timestampFieldMapper().enabled());
assertThat(docMapper.timestampFieldMapper().fieldDataType().getFormat(docMapper.timestampFieldMapper().fieldDataType().getSettings()), equalTo("doc_values"));
}

@Test
public void testMergingConflictsForIndexValues() throws Exception {
List<String> indexValues = new ArrayList<>();
indexValues.add("analyzed");
indexValues.add("no");
indexValues.add("not_analyzed");
String mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("index", indexValues.remove(randomInt(2)))
.endObject()
.endObject().endObject().string();
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();

DocumentMapper docMapper = parser.parse(mapping);
docMapper.refreshSource();
docMapper = parser.parse(docMapper.mappingSource().string());
mapping = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp")
.field("index", indexValues.remove(randomInt(1)))
.endObject()
.endObject().endObject().string();

DocumentMapper.MergeResult mergeResult = docMapper.merge(parser.parse(mapping), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
String[] expectedConflicts = {"mapper [_timestamp] has different index values", "mapper [_timestamp] has different tokenize values"};

for (String conflict : mergeResult.conflicts()) {
assertThat(conflict, isIn(expectedConflicts));
}

}

@Test
public void testMergePaths() throws Exception {
List<String> possiblePathValues = new ArrayList<>();
possiblePathValues.add("some_path");
possiblePathValues.add("anotherPath");
possiblePathValues.add(null);
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
XContentBuilder mapping1 = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp");
String path1 = possiblePathValues.get(randomInt(2));
if (path1!=null) {
mapping1.field("path", path1);
}
mapping1.endObject()
.endObject().endObject();
XContentBuilder mapping2 = XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("_timestamp");
String path2 = possiblePathValues.get(randomInt(2));
if (path2!=null) {
mapping2.field("path", path2);
}
mapping2.endObject()
.endObject().endObject();


testConflict(mapping1.string(), mapping2.string(), parser, (path1==path2?null:"Cannot update path in _timestamp valuev"));

}

void testConflict(String mapping1, String mapping2, DocumentMapperParser parser, String conflict) throws IOException {
DocumentMapper docMapper = parser.parse(mapping1);
docMapper.refreshSource();
docMapper = parser.parse(docMapper.mappingSource().string());
DocumentMapper.MergeResult mergeResult = docMapper.merge(parser.parse(mapping2), DocumentMapper.MergeFlags.mergeFlags().simulate(true));
assertThat(mergeResult.conflicts().length, equalTo(conflict == null ? 0:1));
if (conflict != null) {
assertThat(mergeResult.conflicts()[0], containsString(conflict));
}
}
}
63 changes: 63 additions & 0 deletions src/test/java/org/elasticsearch/index/mapper/update/UpdateMappingOnClusterTests.java
View file
Expand Up @@ -20,13 +20,18 @@
package org.elasticsearch.index.mapper.update;

import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.client.Client;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.MergeMappingException;
import org.elasticsearch.test.ElasticsearchIntegrationTest;
import org.junit.Test;

import java.io.IOException;
import java.util.LinkedHashMap;

import static org.elasticsearch.common.io.Streams.copyToStringFromClasspath;
import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
Expand Down Expand Up @@ -126,4 +131,62 @@ private void compareMappingOnNodes(GetMappingsResponse previousMapping) {
assertThat(previousMapping.getMappings().get(INDEX).get(TYPE).source(), equalTo(currentMapping.getMappings().get(INDEX).get(TYPE).source()));
}
}

@Test
public void testUpdateTimestamp() throws IOException {
XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "lazy").field("format", "doc_values").endObject().field("store", "yes").endObject()
.endObject().endObject();
client().admin().indices().prepareCreate("test").addMapping("type", mapping).get();
GetMappingsResponse appliedMappings = client().admin().indices().prepareGetMappings("test").get();
LinkedHashMap timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("lazy"));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("doc_values"));
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", randomBoolean()).startObject("fielddata").field("loading", "eager").field("format", "array").endObject().field("store", "yes").endObject()
.endObject().endObject();
PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get();
appliedMappings = client().admin().indices().prepareGetMappings("test").get();
timestampMapping = (LinkedHashMap) appliedMappings.getMappings().get("test").get("type").getSourceAsMap().get("_timestamp");
assertThat((Boolean) timestampMapping.get("store"), equalTo(true));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("loading"), equalTo("eager"));
assertThat((String)((LinkedHashMap) timestampMapping.get("fielddata")).get("format"), equalTo("array"));
}

@Test
public void testTimestampMergingConflicts() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject(TYPE)
.startObject("_timestamp").field("enabled", true)
.startObject("fielddata").field("format", "doc_values").endObject()
.field("store", "yes")
.field("index", "analyzed")
.field("path", "foo")
.field("default", "1970-01-01")
.endObject()
.endObject().endObject().string();

client().admin().indices().prepareCreate(INDEX).addMapping(TYPE, mapping).get();

mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", false)
.startObject("fielddata").field("format", "array").endObject()
.field("store", "no")
.field("index", "no")
.field("path", "bar")
.field("default", "1970-01-02")
.endObject()
.endObject().endObject().string();
GetMappingsResponse mappingsBeforeUpdateResponse = client().admin().indices().prepareGetMappings(INDEX).addTypes(TYPE).get();
try {
client().admin().indices().preparePutMapping(INDEX).setType(TYPE).setSource(mapping).get();
fail("This should result in conflicts when merging the mapping");
} catch (MergeMappingException e) {
String[] expectedConflicts = {"mapper [_timestamp] has different index values", "mapper [_timestamp] has different store values", "Cannot update default in _timestamp value. Value is 1970-01-01 now encountering 1970-01-02", "Cannot update path in _timestamp value. Value is foo now encountering bar"};
for (String conflict : expectedConflicts) {
assertThat(e.getDetailedMessage(), containsString(conflict));
}
}
compareMappingOnNodes(mappingsBeforeUpdateResponse);
}
}
2 changes: 1 addition & 1 deletion src/test/java/org/elasticsearch/timestamp/SimpleTimestampTests.java
View file
Expand Up @@ -122,7 +122,7 @@ public void testThatTimestampCanBeSwitchedOnAndOff() throws Exception {
assertTimestampMappingEnabled(index, type, true);

// update some field in the mapping
XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", false).endObject().endObject();
XContentBuilder updateMappingBuilder = jsonBuilder().startObject().startObject("_timestamp").field("enabled", false).field("store", true).endObject().endObject();
PutMappingResponse putMappingResponse = client().admin().indices().preparePutMapping(index).setType(type).setSource(updateMappingBuilder).get();
assertAcked(putMappingResponse);

Expand Down

0 comments on commit 19e3661

Please sign in to comment.