Skip to content

Commit

Permalink
Mappings: Enforce metadata fields are not passed in documents
Browse files Browse the repository at this point in the history 
We previously removed the ability to specify metadata fields inside
documents in elastic#11074, but the backcompat left leniency that allowed this
to still occur. This change locks down parsing so any metadata field
found while parsing a document results in an exception. This only
affects 2.0+ indexes; backcompat is maintained.

closes elastic#13740
  • Loading branch information
@rjernst
rjernst committed Oct 7, 2015
1 parent c0363dd commit 566fef0
Show file tree
Hide file tree
Showing 8 changed files with 89 additions and 16 deletions.
10 changes: 7 additions & 3 deletions core/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
View file
Expand Up @@ -122,7 +122,7 @@ private ParsedDocument innerParseDocument(SourceToParse source) throws MapperPar
// entire type is disabled
parser.skipChildren();
} else if (emptyDoc == false) {
Mapper update = parseObject(context, mapping.root);
Mapper update = parseObject(context, mapping.root, true);
if (update != null) {
context.addDynamicMappingsUpdate(update);
}
Expand Down Expand Up @@ -194,14 +194,18 @@ private ParsedDocument innerParseDocument(SourceToParse source) throws MapperPar
return doc;
}

static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper) throws IOException {
static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper, boolean atRoot) throws IOException {
if (mapper.isEnabled() == false) {
context.parser().skipChildren();
return null;
}
XContentParser parser = context.parser();

String currentFieldName = parser.currentName();
if (atRoot && MapperService.isMetadataField(currentFieldName) &&
Version.indexCreated(context.indexSettings()).onOrAfter(Version.V_2_0_0_beta1)) {
throw new MapperParsingException("Field [" + currentFieldName + "] is a metadata field and cannot be added inside a document. Use the index API request parameters.");
}
XContentParser.Token token = parser.currentToken();
if (token == XContentParser.Token.VALUE_NULL) {
// the object is null ("obj1" : null), simply bail
Expand Down Expand Up @@ -302,7 +306,7 @@ static ObjectMapper parseObject(ParseContext context, ObjectMapper mapper) throw

private static Mapper parseObjectOrField(ParseContext context, Mapper mapper) throws IOException {
if (mapper instanceof ObjectMapper) {
return parseObject(context, (ObjectMapper) mapper);
return parseObject(context, (ObjectMapper) mapper, false);
} else {
FieldMapper fieldMapper = (FieldMapper)mapper;
Mapper update = fieldMapper.parse(context);
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/org/elasticsearch/index/mapper/DynamicMappingTests.java
View file
Expand Up @@ -197,7 +197,7 @@ private Mapper parse(DocumentMapper mapper, DocumentMapperParser parser, XConten
ctx.reset(XContentHelper.createParser(source.source()), new ParseContext.Document(), source);
assertEquals(XContentParser.Token.START_OBJECT, ctx.parser().nextToken());
ctx.parser().nextToken();
return DocumentParser.parseObject(ctx, mapper.root());
return DocumentParser.parseObject(ctx, mapper.root(), true);
}

public void testDynamicMappingsNotNeeded() throws Exception {
Expand Down
14 changes: 14 additions & 0 deletions core/src/test/java/org/elasticsearch/index/mapper/all/SimpleAllMapperTests.java
View file
Expand Up @@ -43,6 +43,7 @@
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParseContext.Document;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.internal.AllFieldMapper;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
Expand Down Expand Up @@ -453,4 +454,17 @@ public void testIncludeInObjectBackcompat() throws Exception {
// the backcompat behavior is actually ignoring directly specifying _all
assertFalse(field.getAllEntries().fields().iterator().hasNext());
}

public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_all", "foo").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_all] is a metadata field and cannot be added inside a document"));
}
}
}
13 changes: 13 additions & 0 deletions core/src/test/java/org/elasticsearch/index/mapper/id/IdMappingTests.java
View file
Expand Up @@ -114,4 +114,17 @@ public void testIncludeInObjectBackcompat() throws Exception {
// _id is not indexed so we need to check _uid
assertEquals(Uid.createUid("type", "1"), doc.rootDoc().get(UidFieldMapper.NAME));
}

public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

try {
docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject().field("_id", "1").endObject().bytes()).type("type"));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_id] is a metadata field and cannot be added inside a document"));
}
}
}
20 changes: 9 additions & 11 deletions core/src/test/java/org/elasticsearch/index/mapper/parent/ParentMappingTests.java
View file
Expand Up @@ -23,6 +23,7 @@
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.index.mapper.Uid;
Expand All @@ -32,21 +33,18 @@

public class ParentMappingTests extends ESSingleNodeTestCase {

public void testParentNotSet() throws Exception {
public void testParentSetInDocNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

ParsedDocument doc = docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject()
.field("_parent", "1122")
.field("x_field", "x_value")
.endObject()
.bytes()).type("type").id("1"));

// no _parent mapping, dynamically used as a string field
assertNull(doc.parent());
assertNotNull(doc.rootDoc().get("_parent"));
try {
docMapper.parse(SourceToParse.source(XContentFactory.jsonBuilder()
.startObject().field("_parent", "1122").endObject().bytes()).type("type").id("1"));
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_parent] is a metadata field and cannot be added inside a document"));
}
}

public void testParentSetInDocBackcompat() throws Exception {
Expand Down
16 changes: 15 additions & 1 deletion core/src/test/java/org/elasticsearch/index/mapper/routing/RoutingTypeMapperTests.java
View file
Expand Up @@ -32,6 +32,7 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.MapperParsingException;
import org.elasticsearch.index.mapper.ParsedDocument;
import org.elasticsearch.index.mapper.SourceToParse;
import org.elasticsearch.test.ESSingleNodeTestCase;
Expand Down Expand Up @@ -113,7 +114,7 @@ public void testIncludeInObjectBackcompat() throws Exception {
Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_1_4_2.id).build();
DocumentMapper docMapper = createIndex("test", settings).mapperService().documentMapperParser().parse(mapping);

XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_timestamp", 2000000).endObject();
XContentBuilder doc = XContentFactory.jsonBuilder().startObject().field("_routing", "foo").endObject();
MappingMetaData mappingMetaData = new MappingMetaData(docMapper);
IndexRequest request = new IndexRequest("test", "type", "1").source(doc);
request.process(MetaData.builder().build(), mappingMetaData, true, "test");
Expand All @@ -122,4 +123,17 @@ public void testIncludeInObjectBackcompat() throws Exception {
assertNull(request.routing());
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_routing"));
}

public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type").endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_routing", "foo").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_routing] is a metadata field and cannot be added inside a document"));
}
}
}
15 changes: 15 additions & 0 deletions core/src/test/java/org/elasticsearch/index/mapper/timestamp/TimestampMappingTests.java
View file
Expand Up @@ -769,6 +769,21 @@ public void testIncludeInObjectBackcompat() throws Exception {
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_timestamp"));
}

public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("default", "1970").field("format", "YYYY").endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_timestamp", 2000000).endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_timestamp] is a metadata field and cannot be added inside a document"));
}
}

public void testThatEpochCanBeIgnoredWithCustomFormat() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_timestamp").field("enabled", true).field("format", "yyyyMMddHH").endObject()
Expand Down
15 changes: 15 additions & 0 deletions core/src/test/java/org/elasticsearch/index/mapper/ttl/TTLMappingTests.java
View file
Expand Up @@ -310,6 +310,21 @@ public void testIncludeInObjectBackcompat() throws Exception {
assertNull(docMapper.parse("test", "type", "1", doc.bytes()).rootDoc().get("_ttl"));
}

public void testIncludeInObjectNotAllowed() throws Exception {
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
.startObject("_ttl").field("enabled", true).endObject()
.endObject().endObject().string();
DocumentMapper docMapper = createIndex("test").mapperService().documentMapperParser().parse(mapping);

try {
docMapper.parse("test", "type", "1", XContentFactory.jsonBuilder()
.startObject().field("_ttl", "2d").endObject().bytes());
fail("Expected failure to parse metadata field");
} catch (MapperParsingException e) {
assertTrue(e.getMessage(), e.getMessage().contains("Field [_ttl] is a metadata field and cannot be added inside a document"));
}
}

private org.elasticsearch.common.xcontent.XContentBuilder getMappingWithTtlEnabled() throws IOException {
return getMappingWithTtlEnabled(null);
}
Expand Down

0 comments on commit 566fef0

Please sign in to comment.