Create aliases for PassThroughObjectMapper subfields in FieldTypeLook…

…up (#106829)

* Alias PassThroughObjectMapper subfields in FieldTypeLookup

* small refactor

* conflict resolution

* add check for circular deps

* use lexicographical ordering for conflicting aliases

* use lexicographical ordering for conflicting aliases

* simplify loop producing aliases

* use priorities for passthrough deduplication

* make priority param required

* add passthrough disclaimer

* small refactor

* small refactor

* revert refactor

* node feature for pass-through priotity

modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TSDBPassthroughIndexingIT.java

@@ -17,6 +17,8 @@
 import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
+import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse;
 import org.elasticsearch.action.get.GetRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -70,6 +72,7 @@ public class TSDBPassthroughIndexingIT extends ESSingleNodeTestCase {
               },
               "attributes": {
                 "type": "passthrough",
+                "priority": 0,
                 "dynamic": true,
                 "time_series_dimension": true
               },
@@ -197,31 +200,20 @@ public void testIndexingGettingAndSearching() throws Exception {
         assertMap(attributes.get("pod.ip"), matchesMap().entry("type", "ip").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.uid"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
         assertMap(attributes.get("pod.name"), matchesMap().entry("type", "keyword").entry("time_series_dimension", true));
-        // alias field mappers:
-        assertMap(
-            ObjectPath.eval("properties.metricset", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.metricset")
-        );
-        assertMap(
-            ObjectPath.eval("properties.number.properties.long", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.number.long")
-        );
-        assertMap(
-            ObjectPath.eval("properties.number.properties.double", mapping),
-            matchesMap().entry("type", "alias").entry("path", "attributes.number.double")
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("name", matchesMap().entry("type", "alias").entry("path", "attributes.pod.name"))
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("uid", matchesMap().entry("type", "alias").entry("path", "attributes.pod.uid"))
-        );
-        assertMap(
-            ObjectPath.eval("properties.pod.properties", mapping),
-            matchesMap().extraOk().entry("ip", matchesMap().entry("type", "alias").entry("path", "attributes.pod.ip"))
-        );
+
+        FieldCapabilitiesResponse fieldCaps = client().fieldCaps(new FieldCapabilitiesRequest().fields("*").indices("k8s")).actionGet();
+        assertTrue(fieldCaps.getField("attributes.metricset").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("metricset").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("attributes.number.long").get("long").isDimension());
+        assertTrue(fieldCaps.getField("number.long").get("long").isDimension());
+        assertTrue(fieldCaps.getField("attributes.number.double").get("float").isDimension());
+        assertTrue(fieldCaps.getField("number.double").get("float").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.ip").get("ip").isDimension());
+        assertTrue(fieldCaps.getField("pod.ip").get("ip").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.uid").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("pod.uid").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("attributes.pod.name").get("keyword").isDimension());
+        assertTrue(fieldCaps.getField("pod.name").get("keyword").isDimension());
     }
 
     public void testIndexingGettingAndSearchingShrunkIndex() throws Exception {

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamGetWriteIndexTests.java

@@ -240,7 +240,7 @@ public void setup() throws Exception {
                 new MetadataFieldMapper[] { dtfm },
                 Collections.emptyMap()
             );
-            MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of(), List.of());
+            MappingLookup mappingLookup = MappingLookup.fromMappers(mapping, List.of(dtfm, dateFieldMapper), List.of());
             indicesService = DataStreamTestHelper.mockIndicesServices(mappingLookup);
         }
 

modules/data-streams/src/test/java/org/elasticsearch/datastreams/DataStreamIndexSettingsProviderTests.java

@@ -631,10 +631,12 @@ public void testGenerateRoutingPathFromPassThroughObject() throws Exception {
                     "properties": {
                         "labels": {
                             "type": "passthrough",
-                            "time_series_dimension": true
+                            "time_series_dimension": true,
+                            "priority": 2
                         },
                         "metrics": {
-                            "type": "passthrough"
+                            "type": "passthrough",
+                            "priority": 1
                         },
                         "another_field": {
                             "type": "keyword"

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -195,8 +195,8 @@ index without timestamp with pipeline:
 ---
 dynamic templates:
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "Support for dynamic fields was added in 8.13"
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -219,6 +219,7 @@ dynamic templates:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  priority: 0
               dynamic_templates:
                 - counter_metric:
                     mapping:
@@ -326,8 +327,8 @@ dynamic templates:
 ---
 dynamic templates - conflicting aliases:
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "Support for dynamic fields was added in 8.13"
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -350,10 +351,12 @@ dynamic templates - conflicting aliases:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  priority: 2
                 resource_attributes:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  priority: 1
               dynamic_templates:
                 - counter_metric:
                     mapping:
@@ -391,7 +394,7 @@ dynamic templates - conflicting aliases:
             filterA:
               filter:
                 term:
-                  dim: "C"
+                  dim: A
               aggs:
                 tsids:
                   terms:
@@ -410,7 +413,7 @@ dynamic templates - conflicting aliases:
             filterA:
               filter:
                 term:
-                  attributes.dim: A
+                  resource_attributes.dim: C
               aggs:
                 tsids:
                   terms:
@@ -423,8 +426,8 @@ dynamic templates - conflicting aliases:
 ---
 dynamic templates with nesting:
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "Support for dynamic fields was added in 8.13"
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -447,13 +450,15 @@ dynamic templates with nesting:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  priority: 2
                 resource:
                   type: object
                   properties:
                     attributes:
                       type: passthrough
                       dynamic: true
                       time_series_dimension: true
+                      priority: 1
               dynamic_templates:
                 - counter_metric:
                     mapping:
@@ -580,8 +585,9 @@ dynamic templates with nesting:
 ---
 dynamic templates with incremental indexing:
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "Support for dynamic fields was added in 8.13"
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
+
   - do:
       allowed_warnings:
         - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
@@ -604,13 +610,15 @@ dynamic templates with incremental indexing:
                   type: passthrough
                   dynamic: true
                   time_series_dimension: true
+                  priority: 2
                 resource:
                   type: object
                   properties:
                     attributes:
                       type: passthrough
                       dynamic: true
                       time_series_dimension: true
+                      priority: 1
               dynamic_templates:
                 - counter_metric:
                     mapping:
@@ -774,8 +782,8 @@ dynamic templates with incremental indexing:
 ---
 subobject in passthrough object auto flatten:
   - requires:
-      cluster_features: ["gte_v8.13.0"]
-      reason: "Support for passthrough fields was added in 8.13"
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
   - do:
       allowed_warnings:
         - "index template [my-passthrough-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-passthrough-template] will take precedence during new index creation"
@@ -794,6 +802,7 @@ subobject in passthrough object auto flatten:
                 attributes:
                   type: passthrough
                   time_series_dimension: true
+                  priority: 0
                   properties:
                     subcategory:
                       type: object
@@ -833,11 +842,36 @@ enable subobjects in passthrough object:
               index:
                 number_of_shards: 1
                 mode: time_series
-                time_series:
-                  start_time: 2023-08-31T13:03:08.138Z
 
             mappings:
               properties:
                 attributes:
                   type: passthrough
                   subobjects: true
+
+---
+passthrough objects with duplicate priority:
+  - requires:
+      cluster_features: ["mapper.pass_through_priority"]
+      reason: support for priority in passthrough objects
+  - do:
+      catch: /has a conflicting param/
+      indices.put_index_template:
+        name: my-dynamic-template
+        body:
+          index_patterns: [k9s*]
+          data_stream: {}
+          template:
+            settings:
+              index:
+                number_of_shards: 1
+                mode: time_series
+
+            mappings:
+              properties:
+                attributes:
+                  type: passthrough
+                  priority: 1
+                resource.attributes:
+                  type: passthrough
+                  priority: 1

server/src/main/java/org/elasticsearch/index/mapper/FieldTypeLookup.java

@@ -14,6 +14,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -38,9 +39,14 @@ final class FieldTypeLookup {
 
     private final int maxParentPathDots;
 
+    FieldTypeLookup(Collection<FieldMapper> fieldMappers, Collection<FieldAliasMapper> fieldAliasMappers) {
+        this(fieldMappers, fieldAliasMappers, List.of(), List.of());
+    }
+
     FieldTypeLookup(
         Collection<FieldMapper> fieldMappers,
         Collection<FieldAliasMapper> fieldAliasMappers,
+        Collection<PassThroughObjectMapper> passThroughMappers,
         Collection<RuntimeField> runtimeFields
     ) {
 
@@ -86,6 +92,35 @@ final class FieldTypeLookup {
             }
         }
 
+        // Pass-though subfields can be referenced without the prefix corresponding to the
+        // PassThroughObjectMapper name. This is achieved by adding a second reference to their
+        // MappedFieldType using the remaining suffix.
+        Map<String, PassThroughObjectMapper> passThroughFieldAliases = new HashMap<>();
+        for (PassThroughObjectMapper passThroughMapper : passThroughMappers) {
+            for (Mapper subfield : passThroughMapper.mappers.values()) {
+                if (subfield instanceof FieldMapper fieldMapper) {
+                    String name = fieldMapper.simpleName();
+                    // Check for conflict between PassThroughObjectMapper subfields.
+                    PassThroughObjectMapper conflict = passThroughFieldAliases.put(name, passThroughMapper);
+                    if (conflict != null) {
+                        if (conflict.priority() > passThroughMapper.priority()) {
+                            // Keep the conflicting field if it has higher priority.
+                            passThroughFieldAliases.put(name, conflict);
+                            continue;
+                        }
+                    } else if (fullNameToFieldType.containsKey(name)) {
+                        // There's an existing field or alias for the same field.
+                        continue;
+                    }
+                    MappedFieldType fieldType = fieldMapper.fieldType();
+                    fullNameToFieldType.put(name, fieldType);
+                    if (fieldType instanceof DynamicFieldType) {
+                        dynamicFieldTypes.put(name, (DynamicFieldType) fieldType);
+                    }
+                }
+            }
+        }
+
         for (MappedFieldType fieldType : RuntimeField.collectFieldTypes(runtimeFields).values()) {
             // this will override concrete fields with runtime fields that have the same name
             fullNameToFieldType.put(fieldType.name(), fieldType);

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -19,6 +19,10 @@
 public class MapperFeatures implements FeatureSpecification {
     @Override
     public Set<NodeFeature> getFeatures() {
-        return Set.of(IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE, RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX);
+        return Set.of(
+            IgnoredSourceFieldMapper.TRACK_IGNORED_SOURCE,
+            PassThroughObjectMapper.PASS_THROUGH_PRIORITY,
+            RangeFieldMapper.NULL_VALUES_OFF_BY_ONE_FIX
+        );
     }
 }