Flatten object mappings when subobjects is false (#103542)

docs/changelog/103542.yaml

@@ -0,0 +1,7 @@
+pr: 103542
+summary: Flatten object mappings when subobjects is false
+area: Mapping
+type: feature
+issues:
+ - 99860
+ - 103497

docs/reference/mapping/params/subobjects.asciidoc

@@ -24,7 +24,12 @@ PUT my-index-000001
     "properties": {
       "metrics": {
         "type":  "object",
-        "subobjects": false <1>
+        "subobjects": false, <1>
+        "properties": {
+          "time": { "type": "long" },
+          "time.min": { "type": "long" },
+          "time.max": { "type": "long" }
+        }
       }
     }
   }
@@ -105,3 +110,70 @@ PUT my-index-000001/_doc/metric_1
 <2> The document does not support objects
 
 The `subobjects` setting for existing fields and the top-level mapping definition cannot be updated.
+
+==== Auto-flattening object mappings
+
+It is generally recommended to define the properties of an object that is configured with `subobjects: false` with dotted field names
+(as shown in the first example).
+However, it is also possible to define these properties as sub-objects in the mappings.
+In that case, the mapping will be automatically flattened before it is stored.
+This makes it easier to re-use existing mappings without having to re-write them.
+
+Note that auto-flattening will not work when certain <<mapping-params, mapping parameters>> are set
+on object mappings that are defined under an object configured with `subobjects: false`:
+
+* The <<enabled, `enabled`>> mapping parameter must not be `false`.
+* The <<dynamic, `dynamic`>> mapping parameter must not contradict the implicit or explicit value of the parent. For example, when `dynamic` is set to `false` in the root of the mapping, object mappers that set `dynamic` to `true` can't be auto-flattened.
+* The <<subobjects, `subobjects`>> mapping parameter must not be set to `true` explicitly.
+
+[source,console]
+--------------------------------------------------
+PUT my-index-000002
+{
+  "mappings": {
+    "properties": {
+      "metrics": {
+        "subobjects": false,
+        "properties": {
+          "time": {
+            "type": "object", <1>
+            "properties": {
+              "min": { "type": "long" }, <2>
+              "max": { "type": "long" }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+GET my-index-000002/_mapping
+--------------------------------------------------
+
+[source,console-result]
+--------------------------------------------------
+{
+  "my-index-000002" : {
+    "mappings" : {
+      "properties" : {
+        "metrics" : {
+          "subobjects" : false,
+          "properties" : {
+            "time.min" : { <3>
+              "type" : "long"
+            },
+            "time.max" : {
+              "type" : "long"
+            }
+          }
+        }
+      }
+    }
+  }
+}
+--------------------------------------------------
+
+<1> The metrics object can contain further object mappings that will be auto-flattened.
+ Object mappings at this level must not set certain mapping parameters as explained above.
+<2> This field will be auto-flattened to `"time.min"` before the mapping is stored.
+<3> The auto-flattened `"time.min"` field can be inspected by looking at the index mapping.

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -556,14 +556,13 @@ dynamic templates with nesting:
   - match: { aggregations.filterA.tsids.buckets.0.doc_count: 2 }
 
 ---
-dynamic templates - subobject in passthrough object error:
+subobject in passthrough object auto flatten:
   - skip:
       version: " - 8.12.99"
-      reason: "Support for dynamic fields was added in 8.13"
+      reason: "Support for passthrough fields was added in 8.13"
   - do:
-      catch: /Tried to add subobject \[subcategory\] to object \[attributes\] which does not support subobjects/
       indices.put_index_template:
-        name: my-dynamic-template
+        name: my-passthrough-template
         body:
           index_patterns: [k9s*]
           data_stream: {}
@@ -576,13 +575,34 @@ dynamic templates - subobject in passthrough object error:
               properties:
                 attributes:
                   type: passthrough
+                  time_series_dimension: true
                   properties:
                     subcategory:
                       type: object
                       properties:
                         dim:
                           type: keyword
+  - do:
+      indices.create_data_stream:
+        name: k9s
+  - is_true: acknowledged
+  # save the backing index names for later use
+  - do:
+      indices.get_data_stream:
+        name: k9s
+  - set: { data_streams.0.indices.0.index_name: idx0name }
 
+  - do:
+      indices.get_mapping:
+        index: $idx0name
+        expand_wildcards: hidden
+  - match: { .$idx0name.mappings.properties.attributes.properties.subcategory\.dim.type: 'keyword' }
+
+---
+enable subobjects in passthrough object:
+  - skip:
+      version: " - 8.12.99"
+      reason: "Support for passthrough fields was added in 8.13"
   - do:
       catch: /Mapping definition for \[attributes\] has unsupported parameters:\  \[subobjects \:\ true\]/
       indices.put_index_template:

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -138,7 +138,7 @@ public PercolatorFieldMapper build(MapperBuilderContext context) {
             PercolatorFieldType fieldType = new PercolatorFieldType(context.buildFullName(name()), meta.getValue());
             // TODO should percolator even allow multifields?
             MultiFields multiFields = multiFieldsBuilder.build(this, context);
-            context = context.createChildContext(name());
+            context = context.createChildContext(name(), null);
             KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(
                 EXTRACTED_TERMS_FIELD_NAME,
                 context,

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -613,7 +613,7 @@ public final MapperBuilderContext createDynamicMapperBuilderContext() {
         if (objectMapper instanceof PassThroughObjectMapper passThroughObjectMapper) {
             containsDimensions = passThroughObjectMapper.containsDimensions();
         }
-        return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, containsDimensions);
+        return new MapperBuilderContext(p, mappingLookup().isSourceSynthetic(), false, containsDimensions, dynamic);
     }
 
     public abstract XContentParser parser();

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -479,7 +479,7 @@ public MultiFields build(Mapper.Builder mainFieldBuilder, MapperBuilderContext c
                     return empty();
                 } else {
                     FieldMapper[] mappers = new FieldMapper[mapperBuilders.size()];
-                    context = context.createChildContext(mainFieldBuilder.name());
+                    context = context.createChildContext(mainFieldBuilder.name(), null);
                     int i = 0;
                     for (Map.Entry<String, Function<MapperBuilderContext, FieldMapper>> entry : this.mapperBuilders.entrySet()) {
                         mappers[i++] = entry.getValue().apply(context);
@@ -1230,7 +1230,7 @@ protected void merge(FieldMapper in, Conflicts conflicts, MapperMergeContext map
             for (Parameter<?> param : getParameters()) {
                 param.merge(in, conflicts);
             }
-            MapperMergeContext childContext = mapperMergeContext.createChildContext(in.simpleName());
+            MapperMergeContext childContext = mapperMergeContext.createChildContext(in.simpleName(), null);
             for (FieldMapper newSubField : in.multiFields.mappers) {
                 multiFieldsBuilder.update(newSubField, childContext);
             }

server/src/main/java/org/elasticsearch/index/mapper/Mapper.java

@@ -24,10 +24,10 @@ public abstract class Mapper implements ToXContentFragment, Iterable<Mapper> {
 
     public abstract static class Builder {
 
-        private final String name;
+        private String name;
 
         protected Builder(String name) {
-            this.name = internFieldName(name);
+            setName(name);
         }
 
         // TODO rename this to leafName?
@@ -37,6 +37,10 @@ public final String name() {
 
         /** Returns a newly built mapper. */
         public abstract Mapper build(MapperBuilderContext context);
+
+        void setName(String name) {
+            this.name = internFieldName(name);
+        }
     }
 
     public interface TypeParser {

server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java

@@ -9,6 +9,9 @@
 package org.elasticsearch.index.mapper;
 
 import org.elasticsearch.common.Strings;
+import org.elasticsearch.core.Nullable;
+
+import java.util.Objects;
 
 /**
  * Holds context for building Mapper objects from their Builders
@@ -19,32 +22,69 @@ public class MapperBuilderContext {
      * The root context, to be used when building a tree of mappers
      */
     public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream) {
-        return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, false);
+        return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, false, ObjectMapper.Defaults.DYNAMIC);
     }
 
     private final String path;
     private final boolean isSourceSynthetic;
     private final boolean isDataStream;
     private final boolean parentObjectContainsDimensions;
+    private final ObjectMapper.Dynamic dynamic;
 
     MapperBuilderContext(String path) {
-        this(path, false, false, false);
+        this(path, false, false, false, ObjectMapper.Defaults.DYNAMIC);
     }
 
-    MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean parentObjectContainsDimensions) {
+    MapperBuilderContext(
+        String path,
+        boolean isSourceSynthetic,
+        boolean isDataStream,
+        boolean parentObjectContainsDimensions,
+        ObjectMapper.Dynamic dynamic
+    ) {
+        Objects.requireNonNull(dynamic, "dynamic must not be null");
         this.path = path;
         this.isSourceSynthetic = isSourceSynthetic;
         this.isDataStream = isDataStream;
         this.parentObjectContainsDimensions = parentObjectContainsDimensions;
+        this.dynamic = dynamic;
+    }
+
+    /**
+     * Creates a new MapperBuilderContext that is a child of this context
+     *
+     * @param name    the name of the child context
+     * @param dynamic strategy for handling dynamic mappings in this context
+     * @return a new MapperBuilderContext with this context as its parent
+     */
+    public MapperBuilderContext createChildContext(String name, @Nullable ObjectMapper.Dynamic dynamic) {
+        return createChildContext(name, this.parentObjectContainsDimensions, dynamic);
     }
 
     /**
      * Creates a new MapperBuilderContext that is a child of this context
-     * @param name the name of the child context
+     *
+     * @param name    the name of the child context
+     * @param dynamic strategy for handling dynamic mappings in this context
+     * @param parentObjectContainsDimensions whether the parent object contains dimensions
      * @return a new MapperBuilderContext with this context as its parent
      */
-    public MapperBuilderContext createChildContext(String name) {
-        return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, parentObjectContainsDimensions);
+    public MapperBuilderContext createChildContext(
+        String name,
+        boolean parentObjectContainsDimensions,
+        @Nullable ObjectMapper.Dynamic dynamic
+    ) {
+        return new MapperBuilderContext(
+            buildFullName(name),
+            this.isSourceSynthetic,
+            this.isDataStream,
+            parentObjectContainsDimensions,
+            getDynamic(dynamic)
+        );
+    }
+
+    protected ObjectMapper.Dynamic getDynamic(@Nullable ObjectMapper.Dynamic dynamic) {
+        return dynamic == null ? this.dynamic : dynamic;
     }
 
     /**
@@ -78,4 +118,7 @@ public boolean parentObjectContainsDimensions() {
         return parentObjectContainsDimensions;
     }
 
+    public ObjectMapper.Dynamic getDynamic() {
+        return dynamic;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java

@@ -46,8 +46,8 @@ public static MapperMergeContext from(MapperBuilderContext mapperBuilderContext,
      * @param name the name of the child context
      * @return a new {@link MapperMergeContext} with this context as its parent
      */
-    MapperMergeContext createChildContext(String name) {
-        return createChildContext(mapperBuilderContext.createChildContext(name));
+    MapperMergeContext createChildContext(String name, ObjectMapper.Dynamic dynamic) {
+        return createChildContext(mapperBuilderContext.createChildContext(name, dynamic));
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/NestedObjectMapper.java

@@ -62,7 +62,11 @@ public NestedObjectMapper build(MapperBuilderContext context) {
                     this.includeInRoot = Explicit.IMPLICIT_FALSE;
                 }
             }
-            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(context.buildFullName(name()), parentIncludedInRoot);
+            NestedMapperBuilderContext nestedContext = new NestedMapperBuilderContext(
+                context.buildFullName(name()),
+                parentIncludedInRoot,
+                context.getDynamic(dynamic)
+            );
             final String fullPath = context.buildFullName(name());
             final String nestedTypePath;
             if (indexCreatedVersion.before(IndexVersions.V_8_0_0)) {
@@ -117,14 +121,14 @@ private static class NestedMapperBuilderContext extends MapperBuilderContext {
 
         final boolean parentIncludedInRoot;
 
-        NestedMapperBuilderContext(String path, boolean parentIncludedInRoot) {
-            super(path);
+        NestedMapperBuilderContext(String path, boolean parentIncludedInRoot, Dynamic dynamic) {
+            super(path, false, false, false, dynamic);
             this.parentIncludedInRoot = parentIncludedInRoot;
         }
 
         @Override
-        public MapperBuilderContext createChildContext(String name) {
-            return new NestedMapperBuilderContext(buildFullName(name), parentIncludedInRoot);
+        public MapperBuilderContext createChildContext(String name, Dynamic dynamic) {
+            return new NestedMapperBuilderContext(buildFullName(name), parentIncludedInRoot, getDynamic(dynamic));
         }
     }
 
@@ -280,7 +284,11 @@ protected MapperMergeContext createChildContext(MapperMergeContext mapperMergeCo
             parentIncludedInRoot |= this.includeInParent.value();
         }
         return mapperMergeContext.createChildContext(
-            new NestedMapperBuilderContext(mapperBuilderContext.buildFullName(name), parentIncludedInRoot)
+            new NestedMapperBuilderContext(
+                mapperBuilderContext.buildFullName(name),
+                parentIncludedInRoot,
+                mapperBuilderContext.getDynamic(dynamic)
+            )
         );
     }