Merge pull request #11074 from rjernst/pr/include-in-object-removal

Mappings: Remove ability to set meta fields inside documents

docs/reference/migration/migrate_2_0.asciidoc

@@ -279,6 +279,11 @@ to provide special features.  They now have limited configuration options.
 * `_field_names` configuration is limited to disabling the field.
 * `_size` configuration is limited to enabling the field.
 
+==== Meta fields in documents
+Meta fields can no longer be specified within a document. They should be specified
+via the API.  For example, instead of adding a field `_parent` within a document,
+use the `parent` url parameter when indexing that document.
+
 ==== Source field limitations
 The `_source` field could previously be disabled dynamically. Since this field
 is a critical piece of many features like the Update API, it is no longer

src/main/java/org/elasticsearch/index/mapper/DocumentMapper.java

@@ -30,6 +30,7 @@
 import org.apache.lucene.search.Filter;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchGenerationException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
@@ -194,6 +195,7 @@ public DocumentMapper(MapperService mapperService, String index, @Nullable Setti
         this.type = rootObjectMapper.name();
         this.typeText = new StringAndBytesText(this.type);
         this.mapping = new Mapping(
+                Version.indexCreated(indexSettings),
                 rootObjectMapper,
                 rootMappers.values().toArray(new RootMapper[rootMappers.values().size()]),
                 sourceTransforms.toArray(new SourceTransform[sourceTransforms.size()]),
@@ -210,7 +212,7 @@ public DocumentMapper(MapperService mapperService, String index, @Nullable Setti
         // collect all the mappers for this type
         List<ObjectMapper> newObjectMappers = new ArrayList<>();
         List<FieldMapper<?>> newFieldMappers = new ArrayList<>();
-        for (RootMapper rootMapper : this.mapping.rootMappersNotIncludedInObject) {
+        for (RootMapper rootMapper : this.mapping.rootMappers) {
             if (rootMapper instanceof FieldMapper) {
                 newFieldMappers.add((FieldMapper) rootMapper);
             }

src/main/java/org/elasticsearch/index/mapper/MapperService.java

@@ -267,8 +267,8 @@ private DocumentMapper merge(DocumentMapper mapper) {
                 List<ObjectMapper> newObjectMappers = new ArrayList<>();
                 List<FieldMapper<?>> newFieldMappers = new ArrayList<>();
                 for (RootMapper rootMapper : mapper.mapping().rootMappers) {
-                    if (!rootMapper.includeInObject() && rootMapper instanceof FieldMapper) {
-                        newFieldMappers.add((FieldMapper<?>) rootMapper);
+                    if (rootMapper instanceof FieldMapper<?>) {
+                        newFieldMappers.add((FieldMapper<?>)rootMapper);
                     }
                 }
                 MapperUtils.collect(mapper.mapping().root, newObjectMappers, newFieldMappers);

src/main/java/org/elasticsearch/index/mapper/Mapping.java

@@ -21,14 +21,16 @@
 
 import com.google.common.collect.ImmutableMap;
 
+import org.elasticsearch.Version;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ToXContent;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.mapper.object.RootObjectMapper;
 
 import java.io.IOException;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
 
@@ -38,6 +40,8 @@
  */
 public final class Mapping implements ToXContent {
 
+    public static final List<String> LEGACY_INCLUDE_IN_OBJECT = Arrays.asList("_all", "_id", "_parent", "_routing", "_timestamp", "_ttl");
+
     /**
      * Transformations to be applied to the source before indexing and/or after loading.
      */
@@ -50,27 +54,31 @@ public interface SourceTransform extends ToXContent {
         Map<String, Object> transformSourceAsMap(Map<String, Object> sourceAsMap);
     }
 
+    final Version indexCreated;
     final RootObjectMapper root;
     final RootMapper[] rootMappers;
-    final RootMapper[] rootMappersNotIncludedInObject;
     final ImmutableMap<Class<? extends RootMapper>, RootMapper> rootMappersMap;
     final SourceTransform[] sourceTransforms;
     volatile ImmutableMap<String, Object> meta;
 
-    public Mapping(RootObjectMapper rootObjectMapper, RootMapper[] rootMappers, SourceTransform[] sourceTransforms, ImmutableMap<String, Object> meta) {
+    public Mapping(Version indexCreated, RootObjectMapper rootObjectMapper, RootMapper[] rootMappers, SourceTransform[] sourceTransforms, ImmutableMap<String, Object> meta) {
+        this.indexCreated = indexCreated;
         this.root = rootObjectMapper;
         this.rootMappers = rootMappers;
-        List<RootMapper> rootMappersNotIncludedInObject = new ArrayList<>();
         ImmutableMap.Builder<Class<? extends RootMapper>, RootMapper> builder = ImmutableMap.builder();
         for (RootMapper rootMapper : rootMappers) {
-            if (rootMapper.includeInObject()) {
+            if (indexCreated.before(Version.V_2_0_0) && LEGACY_INCLUDE_IN_OBJECT.contains(rootMapper.name())) {
                 root.putMapper(rootMapper);
-            } else {
-                rootMappersNotIncludedInObject.add(rootMapper);
             }
             builder.put(rootMapper.getClass(), rootMapper);
         }
-        this.rootMappersNotIncludedInObject = rootMappersNotIncludedInObject.toArray(new RootMapper[rootMappersNotIncludedInObject.size()]);
+        // keep root mappers sorted for consistent serialization
+        Arrays.sort(rootMappers, new Comparator<Mapper>() {
+            @Override
+            public int compare(Mapper o1, Mapper o2) {
+                return o1.name().compareTo(o2.name());
+            }
+        });
         this.rootMappersMap = builder.build();
         this.sourceTransforms = sourceTransforms;
         this.meta = meta;
@@ -85,7 +93,7 @@ public RootObjectMapper root() {
      * Generate a mapping update for the given root object mapper.
      */
     public Mapping mappingUpdate(Mapper rootObjectMapper) {
-        return new Mapping((RootObjectMapper) rootObjectMapper, rootMappers, sourceTransforms, meta);
+        return new Mapping(indexCreated, (RootObjectMapper) rootObjectMapper, rootMappers, sourceTransforms, meta);
     }
 
     /** Get the root mapper with the given class. */
@@ -100,10 +108,6 @@ public void merge(Mapping mergeWith, MergeResult mergeResult) {
 
         root.merge(mergeWith.root, mergeResult);
         for (RootMapper rootMapper : rootMappers) {
-            // root mappers included in root object will get merge in the rootObjectMapper
-            if (rootMapper.includeInObject()) {
-                continue;
-            }
             RootMapper mergeWithRootMapper = mergeWith.rootMapper(rootMapper.getClass());
             if (mergeWithRootMapper != null) {
                 rootMapper.merge(mergeWithRootMapper, mergeResult);
@@ -137,11 +141,12 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
                 if (meta != null && !meta.isEmpty()) {
                     builder.field("_meta", meta);
                 }
+                for (Mapper mapper : rootMappers) {
+                    mapper.toXContent(builder, params);
+                }
                 return builder;
             }
-            // no need to pass here id and boost, since they are added to the root object mapper
-            // in the constructor
-        }, rootMappersNotIncludedInObject);
+        });
         return builder;
     }
 

src/main/java/org/elasticsearch/index/mapper/RootMapper.java

@@ -39,15 +39,4 @@ public interface RootMapper extends Mapper {
      */
     void postParse(ParseContext context) throws IOException;
 
-    /**
-     * Should the mapper be included in the root
-     * {@link org.elasticsearch.index.mapper.object.ObjectMapper}.
-     *
-     * If this method returns true, then {@link #parse(ParseContext)} will be
-     * called if the context has a property that matches the name of this
-     * {@link RootMapper}. Otherwise {@link #parse(ParseContext)} will not
-     * be called.
-     */
-    boolean includeInObject();
-
 }

src/main/java/org/elasticsearch/index/mapper/internal/AllFieldMapper.java

@@ -36,11 +36,10 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.FieldDataType;
-import org.elasticsearch.index.mapper.InternalMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.MergeMappingException;
+import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.RootMapper;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
@@ -61,7 +60,7 @@
 /**
  *
  */
-public class AllFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
+public class AllFieldMapper extends AbstractFieldMapper<String> implements RootMapper {
 
     public interface IncludeInAll extends Mapper {
 
@@ -212,11 +211,6 @@ public Mapper parse(ParseContext context) throws IOException {
         return null;
     }
 
-    @Override
-    public boolean includeInObject() {
-        return true;
-    }
-
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
         if (!enabledState.enabled) {

src/main/java/org/elasticsearch/index/mapper/internal/FieldNamesFieldMapper.java

@@ -22,22 +22,19 @@
 import com.google.common.collect.UnmodifiableIterator;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
-import org.apache.lucene.document.SortedSetDocValuesField;
 import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexableField;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.Version;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.FieldDataType;
-import org.elasticsearch.index.mapper.InternalMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.MergeMappingException;
+import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.RootMapper;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
@@ -58,7 +55,7 @@
  *
  * Added in Elasticsearch 1.3.
  */
-public class FieldNamesFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
+public class FieldNamesFieldMapper extends AbstractFieldMapper<String> implements RootMapper {
 
     public static final String NAME = "_field_names";
 
@@ -188,11 +185,6 @@ public Mapper parse(ParseContext context) throws IOException {
         return null;
     }
 
-    @Override
-    public boolean includeInObject() {
-        return false;
-    }
-
     static Iterable<String> extractFieldNames(final String fullPath) {
         return new Iterable<String>() {
             @Override

src/main/java/org/elasticsearch/index/mapper/internal/IdFieldMapper.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.index.mapper.internal;
 
 import com.google.common.collect.Iterables;
-
 import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
@@ -44,7 +43,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.fielddata.FieldDataType;
-import org.elasticsearch.index.mapper.InternalMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MergeMappingException;
@@ -57,7 +55,6 @@
 
 import java.io.IOException;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -68,7 +65,7 @@
 /**
  * 
  */
-public class IdFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
+public class IdFieldMapper extends AbstractFieldMapper<String> implements RootMapper {
 
     public static final String NAME = "_id";
 
@@ -264,11 +261,6 @@ public void postParse(ParseContext context) throws IOException {
         // it either get built in the preParse phase, or get parsed...
     }
 
-    @Override
-    public boolean includeInObject() {
-        return true;
-    }
-
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
         XContentParser parser = context.parser();

src/main/java/org/elasticsearch/index/mapper/internal/IndexFieldMapper.java

@@ -30,12 +30,11 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.FieldDataType;
-import org.elasticsearch.index.mapper.InternalMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperBuilders;
 import org.elasticsearch.index.mapper.MapperParsingException;
-import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.MergeMappingException;
+import org.elasticsearch.index.mapper.MergeResult;
 import org.elasticsearch.index.mapper.ParseContext;
 import org.elasticsearch.index.mapper.RootMapper;
 import org.elasticsearch.index.mapper.core.AbstractFieldMapper;
@@ -51,7 +50,7 @@
 /**
  *
  */
-public class IndexFieldMapper extends AbstractFieldMapper<String> implements InternalMapper, RootMapper {
+public class IndexFieldMapper extends AbstractFieldMapper<String> implements RootMapper {
 
     public static final String NAME = "_index";
 
@@ -170,11 +169,6 @@ public Mapper parse(ParseContext context) throws IOException {
         return null;
     }
 
-    @Override
-    public boolean includeInObject() {
-        return false;
-    }
-
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
         if (!enabledState.enabled) {

src/main/java/org/elasticsearch/index/mapper/internal/ParentFieldMapper.java

@@ -19,7 +19,6 @@
 package org.elasticsearch.index.mapper.internal;
 
 import com.google.common.base.Objects;
-
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
@@ -36,7 +35,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.index.fielddata.FieldDataType;
 import org.elasticsearch.index.mapper.DocumentMapper;
-import org.elasticsearch.index.mapper.InternalMapper;
 import org.elasticsearch.index.mapper.Mapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
 import org.elasticsearch.index.mapper.MergeMappingException;
@@ -62,7 +60,7 @@
 /**
  *
  */
-public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements InternalMapper, RootMapper {
+public class ParentFieldMapper extends AbstractFieldMapper<Uid> implements RootMapper {
 
     public static final String NAME = "_parent";
 
@@ -181,11 +179,6 @@ public void postParse(ParseContext context) throws IOException {
         parse(context);
     }
 
-    @Override
-    public boolean includeInObject() {
-        return true;
-    }
-
     @Override
     protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
         if (!active()) {