Add ignored field values to synthetic source

docs/changelog/107567.yaml

@@ -0,0 +1,5 @@
+pr: 107567
+summary: Add ignored field values to synthetic source
+area: Mapping
+type: enhancement
+issues: []

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/20_synthetic_source.yml

@@ -36,3 +36,154 @@ nested is disabled:
                 properties:
                   foo:
                     type: keyword
+
+---
+object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_values"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 1
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              name:
+                type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "name": "aaaa", "some_string": "AaAa", "some_int": 1000, "some_double": 123.456789, "some_bool": true, "a.very.deeply.nested.field": "AAAA" }'
+          - '{ "create": { } }'
+          - '{ "name": "bbbb", "some_string": "BbBb", "some_int": 2000, "some_double": 321.987654, "some_bool": false, "a.very.deeply.nested.field": "BBBB" }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.name: aaaa }
+  - match: { hits.hits.0._source.some_string: AaAa }
+  - match: { hits.hits.0._source.some_int: 1000 }
+  - match: { hits.hits.0._source.some_double: 123.456789 }
+  - match: { hits.hits.0._source.a.very.deeply.nested.field: AAAA }
+  - match: { hits.hits.0._source.some_bool: true }
+  - match: { hits.hits.1._source.name: bbbb }
+  - match: { hits.hits.1._source.some_string: BbBb }
+  - match: { hits.hits.1._source.some_int: 2000 }
+  - match: { hits.hits.1._source.some_double: 321.987654 }
+  - match: { hits.hits.1._source.a.very.deeply.nested.field: BBBB }
+
+
+---
+nested object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_values"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 3
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              path:
+                properties:
+                  to:
+                    properties:
+                      name:
+                        type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "path.to.name": "aaaa", "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
+          - '{ "create": { } }'
+          - '{ "path.to.name": "bbbb", "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.path.to.name: aaaa }
+  - match: { hits.hits.0._source.path.to.surname: AaAa }
+  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.1._source.path.to.name: bbbb }
+  - match: { hits.hits.1._source.path.to.surname: BbBb }
+  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }
+
+
+---
+empty object with unmapped fields:
+  - requires:
+      cluster_features: ["mapper.track_ignored_values"]
+      reason: requires tracking ignored values
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index:
+              mapping:
+                total_fields:
+                  ignore_dynamic_beyond_limit: true
+                  limit: 3
+
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              path:
+                properties:
+                  to:
+                    properties:
+                      name:
+                        type: keyword
+
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{ "create": { } }'
+          - '{ "path.to.surname": "AaAa", "path.some.other.name": "AaAaAa"  }'
+          - '{ "create": { } }'
+          - '{ "path.to.surname": "BbBb", "path.some.other.name": "BbBbBb"  }'
+
+  - do:
+      search:
+        index: test
+
+  - match: { hits.total.value: 2 }
+  - match: { hits.hits.0._source.path.to.surname: AaAa }
+  - match: { hits.hits.0._source.path.some.other.name: AaAaAa }
+  - match: { hits.hits.1._source.path.to.surname: BbBb }
+  - match: { hits.hits.1._source.path.some.other.name: BbBbBb }

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/nodes.stats/11_indices_metrics.yml

@@ -417,8 +417,8 @@
 
   - requires:
       test_runner_features: [arbitrary_key]
-      cluster_features: ["gte_v8.5.0"]
-      reason:  "mappings added in version 8.5.0"
+      cluster_features: ["mapper.track_ignored_values"]
+      reason:  "_ignored_values added to mappings"
 
   - do:
       indices.create:
@@ -476,35 +476,36 @@
   # 4. _field_names
   # 5. _id
   # 6. _ignored
-  # 7. _index
-  # 8. _nested_path
-  # 9. _routing
-  # 10. _seq_no
-  # 11. _source
-  # 12. _tier
-  # 13. _version
-  # 14. @timestamp
-  # 15. authors.age
-  # 16. authors.company
-  # 17. authors.company.keyword
-  # 18. authors.name.last_name
-  # 19. authors.name.first_name
-  # 20. authors.name.full_name
-  # 21. link
-  # 22. title
-  # 23. url
+  # 7. _ignored_values
+  # 8. _index
+  # 9. _nested_path
+  # 10. _routing
+  # 11. _seq_no
+  # 12. _source
+  # 13. _tier
+  # 14. _version
+  # 15. @timestamp
+  # 16. authors.age
+  # 17. authors.company
+  # 18. authors.company.keyword
+  # 19. authors.name.last_name
+  # 20. authors.name.first_name
+  # 21. authors.name.full_name
+  # 22. link
+  # 23. title
+  # 24. url
   # Object mappers:
-  # 24. authors
-  # 25. authors.name
+  # 25. authors
+  # 26. authors.name
   # Runtime field mappers:
-  # 26. a_source_field
+  # 27. a_source_field
 
-  - gte: { nodes.$node_id.indices.mappings.total_count: 26 }
+  - gte: { nodes.$node_id.indices.mappings.total_count: 27 }
   - is_true: nodes.$node_id.indices.mappings.total_estimated_overhead
   - gte: { nodes.$node_id.indices.mappings.total_estimated_overhead_in_bytes: 26624 }
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 26 }
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_count: 27 }
   - is_true: nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead
-  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 26624 }
+  - match: { nodes.$node_id.indices.indices.index1.mappings.total_estimated_overhead_in_bytes: 27648 }
 
 ---
 "indices mappings does not exist in shards level":

server/src/main/java/module-info.java

@@ -424,6 +424,7 @@
             org.elasticsearch.rest.RestFeatures,
             org.elasticsearch.indices.IndicesFeatures,
             org.elasticsearch.action.admin.cluster.allocation.AllocationStatsFeatures,
+            org.elasticsearch.index.mapper.MapperFeatures,
             org.elasticsearch.search.retriever.RetrieversFeatures;
 
     uses org.elasticsearch.plugins.internal.SettingsExtension;

server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java

@@ -104,6 +104,7 @@ public int get() {
     private final MappingParserContext mappingParserContext;
     private final SourceToParse sourceToParse;
     private final Set<String> ignoredFields;
+    private final List<IgnoredValuesFieldMapper.Values> ignoredFieldValues;
     private final Map<String, List<Mapper>> dynamicMappers;
     private final DynamicMapperSize dynamicMappersSize;
     private final Map<String, ObjectMapper> dynamicObjectMappers;
@@ -122,6 +123,7 @@ private DocumentParserContext(
         MappingParserContext mappingParserContext,
         SourceToParse sourceToParse,
         Set<String> ignoreFields,
+        List<IgnoredValuesFieldMapper.Values> ignoredFieldValues,
         Map<String, List<Mapper>> dynamicMappers,
         Map<String, ObjectMapper> dynamicObjectMappers,
         Map<String, List<RuntimeField>> dynamicRuntimeFields,
@@ -139,6 +141,7 @@ private DocumentParserContext(
         this.mappingParserContext = mappingParserContext;
         this.sourceToParse = sourceToParse;
         this.ignoredFields = ignoreFields;
+        this.ignoredFieldValues = ignoredFieldValues;
         this.dynamicMappers = dynamicMappers;
         this.dynamicObjectMappers = dynamicObjectMappers;
         this.dynamicRuntimeFields = dynamicRuntimeFields;
@@ -159,6 +162,7 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic,
             in.mappingParserContext,
             in.sourceToParse,
             in.ignoredFields,
+            in.ignoredFieldValues,
             in.dynamicMappers,
             in.dynamicObjectMappers,
             in.dynamicRuntimeFields,
@@ -186,6 +190,7 @@ protected DocumentParserContext(
             mappingParserContext,
             source,
             new HashSet<>(),
+            new ArrayList<>(),
             new HashMap<>(),
             new HashMap<>(),
             new HashMap<>(),
@@ -251,6 +256,20 @@ public final Collection<String> getIgnoredFields() {
         return Collections.unmodifiableCollection(ignoredFields);
     }
 
+    /**
+     * Add the given ignored values to the corresponding list.
+     */
+    public final void addIgnoredValues(IgnoredValuesFieldMapper.Values values) {
+        ignoredFieldValues.add(values);
+    }
+
+    /**
+     * Return the collection of values for fields that have been ignored so far.
+     */
+    public final Collection<IgnoredValuesFieldMapper.Values> getIgnoredFieldValues() {
+        return Collections.unmodifiableCollection(ignoredFieldValues);
+    }
+
     /**
      * Add the given {@code field} to the _field_names field
      *
@@ -345,6 +364,16 @@ public final boolean addDynamicMapper(Mapper mapper) {
             int additionalFieldsToAdd = getNewFieldsSize() + mapperSize;
             if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) {
                 if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) {
+                    if (indexSettings().getMode().isSyntheticSourceEnabled() || mappingLookup.isSourceSynthetic()) {
+                        try {
+                            int parentOffset = parent() instanceof RootObjectMapper ? 0 : parent().fullPath().length() + 1;
+                            addIgnoredValues(
+                                new IgnoredValuesFieldMapper.Values(mapper.name(), parentOffset, FieldDataParseHelper.encodeToken(parser()))
+                            );
+                        } catch (IOException e) {
+                            throw new IllegalArgumentException("failed to parse field [" + mapper.name() + " ]", e);
+                        }
+                    }
                     addIgnoredField(mapper.name());
                     return false;
                 }