From f642b8a3aa798b13dd66b9b8f9ac4e2bd8d1f4e8 Mon Sep 17 00:00:00 2001 From: Felix Barnsteiner Date: Fri, 2 Feb 2024 11:53:52 +0100 Subject: [PATCH] Add setting to ignore dynamic fields when field limit is reached (#96235) Adds a new `index.mapping.total_fields.ignore_dynamic_beyond_limit` index setting. When set to `true`, new fields are added to the mapping as long as the field limit (`index.mapping.total_fields.limit`) is not exceeded. Fields that would exceed the limit are not added to the mapping, similar to `dynamic: false`. Ignored fields are added to the `_ignored` metadata field. Relates to https://github.com/elastic/elasticsearch/issues/89911 To make this easier to review, this is split into the following PRs: - [x] https://github.com/elastic/elasticsearch/pull/102915 - [x] https://github.com/elastic/elasticsearch/pull/102936 - [x] https://github.com/elastic/elasticsearch/pull/104769 Related but not a prerequisite: - [ ] https://github.com/elastic/elasticsearch/pull/102885 --- docs/changelog/96235.yaml | 5 + .../mapping/fields/ignored-field.asciidoc | 7 +- .../mapping/mapping-settings-limit.asciidoc | 11 +- .../reference/mapping/params/dynamic.asciidoc | 7 + .../common-issues/mapping-explosion.asciidoc | 5 +- .../index/mapper/DynamicMappingIT.java | 246 ++++++++++++++++-- .../metadata/MetadataMappingService.java | 3 +- .../common/settings/IndexScopedSettings.java | 1 + .../elasticsearch/index/IndexSettings.java | 15 ++ .../elasticsearch/index/engine/Engine.java | 5 - .../index/mapper/DocumentParser.java | 15 +- .../index/mapper/DocumentParserContext.java | 98 +++++-- .../index/mapper/DynamicFieldsBuilder.java | 125 ++++----- .../index/mapper/MapperMergeContext.java | 2 - .../index/mapper/MapperService.java | 45 +++- .../index/mapper/MappingLookup.java | 6 +- .../index/mapper/ParsedDocument.java | 1 - .../index/mapper/RootObjectMapper.java | 6 +- .../mapper/DocumentParserContextTests.java | 70 +++++ .../index/mapper/DocumentParserTests.java | 7 +- .../index/mapper/MapperServiceTests.java | 226 ++++++++++++++++ .../mapper/TestDocumentParserContext.java | 12 +- .../TransportResumeFollowActionTests.java | 1 + .../mapper/ConstantKeywordFieldMapper.java | 4 +- 24 files changed, 795 insertions(+), 128 deletions(-) create mode 100644 docs/changelog/96235.yaml create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java diff --git a/docs/changelog/96235.yaml b/docs/changelog/96235.yaml new file mode 100644 index 0000000000000..83d1eaf74916b --- /dev/null +++ b/docs/changelog/96235.yaml @@ -0,0 +1,5 @@ +pr: 96235 +summary: Add `index.mapping.total_fields.ignore_dynamic_beyond_limit` setting to ignore dynamic fields when field limit is reached +area: Mapping +type: enhancement +issues: [] diff --git a/docs/reference/mapping/fields/ignored-field.asciidoc b/docs/reference/mapping/fields/ignored-field.asciidoc index 5249d2d379a8e..5fd6c478438ab 100644 --- a/docs/reference/mapping/fields/ignored-field.asciidoc +++ b/docs/reference/mapping/fields/ignored-field.asciidoc @@ -4,8 +4,11 @@ The `_ignored` field indexes and stores the names of every field in a document that has been ignored when the document was indexed. This can, for example, be the case when the field was malformed and <> -was turned on, or when a `keyword` fields value exceeds its optional -<> setting. +was turned on, when a `keyword` field's value exceeds its optional +<> setting, or when +<> has been reached and +<> +is set to `true`. This field is searchable with <>, <> and <> diff --git a/docs/reference/mapping/mapping-settings-limit.asciidoc b/docs/reference/mapping/mapping-settings-limit.asciidoc index c499ca7675f2c..6e05e6ea60855 100644 --- a/docs/reference/mapping/mapping-settings-limit.asciidoc +++ b/docs/reference/mapping/mapping-settings-limit.asciidoc @@ -20,9 +20,18 @@ limits the maximum number of clauses in a query. + [TIP] ==== -If your field mappings contain a large, arbitrary set of keys, consider using the <> data type. +If your field mappings contain a large, arbitrary set of keys, consider using the <> data type, +or setting the index setting `index.mapping.total_fields.ignore_dynamic_beyond_limit` to `true`. ==== +`index.mapping.total_fields.ignore_dynamic_beyond_limit`:: + This setting determines what happens when a dynamically mapped field would exceed the total fields limit. + When set to `false` (the default), the index request of the document that tries to add a dynamic field to the mapping will fail with the message `Limit of total fields [X] has been exceeded`. + When set to `true`, the index request will not fail. + Instead, fields that would exceed the limit are not added to the mapping, similar to <>. + The fields that were not added to the mapping will be added to the <>. + The default value is `false`. + `index.mapping.depth.limit`:: The maximum depth for a field, which is measured as the number of inner objects. For instance, if all fields are defined at the root object level, diff --git a/docs/reference/mapping/params/dynamic.asciidoc b/docs/reference/mapping/params/dynamic.asciidoc index ac95d5de80b89..094828fb445dd 100644 --- a/docs/reference/mapping/params/dynamic.asciidoc +++ b/docs/reference/mapping/params/dynamic.asciidoc @@ -90,3 +90,10 @@ accepts the following parameters: to the mapping, and new fields must be added explicitly. `strict`:: If new fields are detected, an exception is thrown and the document is rejected. New fields must be explicitly added to the mapping. + +[[dynamic-field-limit]] +==== Behavior when reaching the field limit +Setting `dynamic` to either `true` or `runtime` will only add dynamic fields until <> is reached. +By default, index requests for documents that would exceed the field limit will fail, +unless <> is set to `true`. +In that case, ignored fields are added to the <>. diff --git a/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc b/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc index 48e9f802e13f8..5ba18df3e6a6b 100644 --- a/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc +++ b/docs/reference/troubleshooting/common-issues/mapping-explosion.asciidoc @@ -41,13 +41,16 @@ timing out in the browser's Developer Tools Network tab. doesn't normally cause problems unless it's combined with overriding <>. The default `1000` limit is considered generous, though overriding to `10000` -doesn't cause noticable impact depending on use case. However, to give +doesn't cause noticeable impact depending on use case. However, to give a bad example, overriding to `100000` and this limit being hit by mapping totals would usually have strong performance implications. If your index mapped fields expect to contain a large, arbitrary set of keys, you may instead consider: +* Setting <> to `true`. +Instead of rejecting documents that exceed the field limit, this will ignore dynamic fields once the limit is reached. + * Using the <> data type. Please note, however, that flattened objects is link:https://github.com/elastic/kibana/issues/25820[not fully supported in {kib}] yet. For example, this could apply to sub-mappings like { `host.name` , `host.os`, `host.version` }. Desired fields are still accessed by diff --git a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java index 4c1c564bdc734..76d305ce8ea4b 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/index/mapper/DynamicMappingIT.java @@ -20,12 +20,15 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Randomness; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.query.GeoBoundingBoxQueryBuilder; +import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.MatchQueryBuilder; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.script.field.WriteField; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.xcontent.XContentBuilder; @@ -35,21 +38,29 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.vectors.DenseVectorFieldMapper.MIN_DIMS_FOR_DYNAMIC_FLOAT_MAPPING; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertResponse; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchHits; +import static org.hamcrest.Matchers.aMapWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasKey; import static org.hamcrest.Matchers.instanceOf; public class DynamicMappingIT extends ESIntegTestCase { @@ -77,6 +88,16 @@ public void testConflictingDynamicMappings() { } } + public void testSimpleDynamicMappingsSuccessful() { + createIndex("index"); + client().prepareIndex("index").setId("1").setSource("a.x", 1).get(); + client().prepareIndex("index").setId("2").setSource("a.y", 2).get(); + + Map mappings = indicesAdmin().prepareGetMappings("index").get().mappings().get("index").sourceAsMap(); + assertTrue(new WriteField("properties.a", () -> mappings).exists()); + assertTrue(new WriteField("properties.a.properties.x", () -> mappings).exists()); + } + public void testConflictingDynamicMappingsBulk() { // we don't use indexRandom because the order of requests is important here createIndex("index"); @@ -87,18 +108,60 @@ public void testConflictingDynamicMappingsBulk() { assertTrue(bulkResponse.hasFailures()); } - private static void assertMappingsHaveField(GetMappingsResponse mappings, String index, String field) throws IOException { - MappingMetadata indexMappings = mappings.getMappings().get("index"); - assertNotNull(indexMappings); - Map typeMappingsMap = indexMappings.getSourceAsMap(); - @SuppressWarnings("unchecked") - Map properties = (Map) typeMappingsMap.get("properties"); - assertTrue("Could not find [" + field + "] in " + typeMappingsMap.toString(), properties.containsKey(field)); + public void testArrayWithDifferentTypes() { + createIndex("index"); + BulkResponse bulkResponse = client().prepareBulk() + .add(client().prepareIndex("index").setId("1").setSource("foo", List.of(42, "bar"))) + .get(); + + assertTrue(bulkResponse.hasFailures()); + assertEquals( + "mapper [foo] cannot be changed from type [long] to [text]", + bulkResponse.getItems()[0].getFailure().getCause().getMessage() + ); + } + + public void testArraysCountAsOneTowardsFieldLimit() { + createIndex("index", Settings.builder().put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2).build()); + BulkResponse bulkResponse = client().prepareBulk() + .add(client().prepareIndex("index").setId("1").setSource("field1", List.of(1, 2), "field2", 1)) + .get(); + + assertFalse(bulkResponse.hasFailures()); } public void testConcurrentDynamicUpdates() throws Throwable { - createIndex("index"); - final Thread[] indexThreads = new Thread[32]; + int numberOfFieldsToCreate = 32; + Map properties = indexConcurrently(numberOfFieldsToCreate, Settings.builder()); + assertThat(properties, aMapWithSize(numberOfFieldsToCreate)); + for (int i = 0; i < numberOfFieldsToCreate; i++) { + assertThat(properties, hasKey("field" + i)); + } + } + + public void testConcurrentDynamicIgnoreBeyondLimitUpdates() throws Throwable { + int numberOfFieldsToCreate = 32; + Map properties = indexConcurrently( + numberOfFieldsToCreate, + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfFieldsToCreate) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + ); + // every field is a multi-field (text + keyword) + assertThat(properties, aMapWithSize(16)); + assertResponse( + prepareSearch("index").setQuery(new MatchAllQueryBuilder()).setSize(numberOfFieldsToCreate).addFetchField("*"), + response -> { + long ignoredFields = Arrays.stream(response.getHits().getHits()).filter(hit -> hit.field("_ignored") != null).count(); + assertEquals(16, ignoredFields); + } + ); + } + + private Map indexConcurrently(int numberOfFieldsToCreate, Settings.Builder settings) throws Throwable { + indicesAdmin().prepareCreate("index").setSettings(settings).get(); + ensureGreen("index"); + final Thread[] indexThreads = new Thread[numberOfFieldsToCreate]; final CountDownLatch startLatch = new CountDownLatch(1); final AtomicReference error = new AtomicReference<>(); for (int i = 0; i < indexThreads.length; ++i) { @@ -126,14 +189,17 @@ public void run() { if (error.get() != null) { throw error.get(); } - Thread.sleep(2000); - GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); - for (int i = 0; i < indexThreads.length; ++i) { - assertMappingsHaveField(mappings, "index", "field" + i); - } - for (int i = 0; i < indexThreads.length; ++i) { + client().admin().indices().prepareRefresh("index").get(); + for (int i = 0; i < numberOfFieldsToCreate; ++i) { assertTrue(client().prepareGet("index", Integer.toString(i)).get().isExists()); } + GetMappingsResponse mappings = indicesAdmin().prepareGetMappings("index").get(); + MappingMetadata indexMappings = mappings.getMappings().get("index"); + assertNotNull(indexMappings); + Map typeMappingsMap = indexMappings.getSourceAsMap(); + @SuppressWarnings("unchecked") + Map properties = (Map) typeMappingsMap.get("properties"); + return properties; } public void testPreflightCheckAvoidsMaster() throws InterruptedException, IOException { @@ -221,15 +287,157 @@ public void onFailure(Exception e) { Exception e = expectThrows(DocumentParsingException.class, prepareIndex("index").setId("2").setSource("field2", "value2")); assertThat(e.getMessage(), Matchers.containsString("failed to parse")); assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); - assertThat( - e.getCause().getMessage(), - Matchers.containsString("Limit of total fields [2] has been exceeded while adding new fields [1]") - ); + assertThat(e.getCause().getMessage(), Matchers.containsString("Limit of total fields [2] has been exceeded")); } finally { indexingCompletedLatch.countDown(); } } + public void testIgnoreDynamicBeyondLimitSingleMultiField() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", "text"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("_ignored"))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field1"))); + }); + } + + public void testIgnoreDynamicBeyondLimitMultiField() { + indexIgnoreDynamicBeyond(2, orderedMap("field1", 1, "field2", "text"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testIgnoreDynamicArrayField() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", 1, "field2", List.of(1, 2)), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectField() { + indexIgnoreDynamicBeyond(3, orderedMap("a.b", 1, "a.c", 2, "a.d", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectField2() { + indexIgnoreDynamicBeyond(3, orderedMap("a.b", 1, "a.c", 2, "b.a", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("b"))); + }); + } + + public void testIgnoreDynamicBeyondLimitDottedObjectMultiField() { + indexIgnoreDynamicBeyond(4, orderedMap("a.b", "foo", "a.c", 2, "a.d", 3), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.b.keyword", "a.c", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.b.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of(2L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitObjectMultiField() { + indexIgnoreDynamicBeyond(5, orderedMap("a", orderedMap("b", "foo", "c", "bar", "d", 3)), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("a.b", "a.b.keyword", "a.c", "a.c.keyword", "_ignored"))); + assertThat(fields.get("a.b").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.b.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("a.c").getValues(), equalTo(List.of("bar"))); + assertThat(fields.get("a.c.keyword").getValues(), equalTo(List.of("bar"))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("a.d"))); + }); + } + + public void testIgnoreDynamicBeyondLimitRuntimeFields() { + indexIgnoreDynamicBeyond(1, orderedMap("field1", 1, "field2", List.of(1, 2)), Map.of("dynamic", "runtime"), fields -> { + assertThat(fields.keySet(), equalTo(Set.of("field1", "_ignored"))); + assertThat(fields.get("field1").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("field2"))); + }); + } + + public void testFieldLimitRuntimeAndDynamic() throws Exception { + assertAcked( + indicesAdmin().prepareCreate("test") + .setSettings( + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 5) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + ) + .setMapping(""" + { + "dynamic": "runtime", + "properties": { + "runtime": { + "type": "object" + }, + "mapped_obj": { + "type": "object", + "dynamic": "true" + } + } + }""") + .get() + ); + + client().index( + new IndexRequest("test").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(orderedMap("dynamic.keyword", "foo", "mapped_obj.number", 1, "mapped_obj.string", "foo")) + ).get(); + + assertResponse(prepareSearch("test").setQuery(new MatchAllQueryBuilder()).addFetchField("*"), r -> { + var fields = r.getHits().getHits()[0].getFields(); + assertThat(fields.keySet(), equalTo(Set.of("dynamic.keyword", "mapped_obj.number", "_ignored"))); + assertThat(fields.get("dynamic.keyword").getValues(), equalTo(List.of("foo"))); + assertThat(fields.get("mapped_obj.number").getValues(), equalTo(List.of(1L))); + assertThat(fields.get("_ignored").getValues(), equalTo(List.of("mapped_obj.string"))); + }); + } + + private LinkedHashMap orderedMap(Object... entries) { + var map = new LinkedHashMap(); + for (int i = 0; i < entries.length; i += 2) { + map.put((String) entries[i], entries[i + 1]); + } + return map; + } + + private void indexIgnoreDynamicBeyond(int fieldLimit, Map source, Consumer> fieldsConsumer) { + indexIgnoreDynamicBeyond(fieldLimit, source, Map.of(), fieldsConsumer); + } + + private void indexIgnoreDynamicBeyond( + int fieldLimit, + Map source, + Map mapping, + Consumer> fieldsConsumer + ) { + client().admin() + .indices() + .prepareCreate("index") + .setSettings( + Settings.builder() + .put(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), fieldLimit) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build() + ) + .setMapping(mapping) + .get(); + ensureGreen("index"); + client().prepareIndex("index").setId("1").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).setSource(source).get(); + assertResponse( + prepareSearch("index").setQuery(new MatchAllQueryBuilder()).addFetchField("*"), + r -> fieldsConsumer.accept(r.getHits().getHits()[0].getFields()) + ); + } + public void testTotalFieldsLimitWithRuntimeFields() { Settings indexSettings = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java index 6307ed768e813..7e2c0849a6fad 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataMappingService.java @@ -151,7 +151,8 @@ private static ClusterState applyRequest( MapperService.mergeMappings( mapperService.documentMapper(), mapping, - request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE + request.autoUpdate() ? MergeReason.MAPPING_AUTO_UPDATE : MergeReason.MAPPING_UPDATE, + mapperService.getIndexSettings() ); } Metadata.Builder builder = Metadata.builder(metadata); diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index c1b8d51c255db..41dd840b0c0e7 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -151,6 +151,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { Store.INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING, MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, + MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 83a6d9319c75a..6decd20e0a41f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -42,6 +42,7 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; @@ -753,6 +754,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingNestedFieldsLimit; private volatile long mappingNestedDocsLimit; private volatile long mappingTotalFieldsLimit; + private volatile boolean ignoreDynamicFieldsBeyondLimit; private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; @@ -897,6 +899,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingNestedFieldsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); mappingNestedDocsLimit = scopedSettings.get(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); mappingTotalFieldsLimit = scopedSettings.get(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + ignoreDynamicFieldsBeyondLimit = scopedSettings.get(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); @@ -976,6 +979,10 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti scopedSettings.addSettingsUpdateConsumer(INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING, this::setRetentionLeaseMillis); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING, this::setMappingNestedFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING, this::setMappingNestedDocsLimit); + scopedSettings.addSettingsUpdateConsumer( + INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING, + this::setIgnoreDynamicFieldsBeyondLimit + ); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING, this::setMappingTotalFieldsLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_DEPTH_LIMIT_SETTING, this::setMappingDepthLimit); scopedSettings.addSettingsUpdateConsumer(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, this::setMappingFieldNameLengthLimit); @@ -1519,6 +1526,14 @@ private void setMappingTotalFieldsLimit(long value) { this.mappingTotalFieldsLimit = value; } + private void setIgnoreDynamicFieldsBeyondLimit(boolean ignoreDynamicFieldsBeyondLimit) { + this.ignoreDynamicFieldsBeyondLimit = ignoreDynamicFieldsBeyondLimit; + } + + public boolean isIgnoreDynamicFieldsBeyondLimit() { + return ignoreDynamicFieldsBeyondLimit; + } + public long getMappingDepthLimit() { return mappingDepthLimit; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 3849095a94e6e..a910e496ce1b5 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -629,11 +629,6 @@ public DeleteResult(Exception failure, long version, long term, long seqNo, bool this.found = found; } - public DeleteResult(Mapping requiredMappingUpdate, String id) { - super(Operation.TYPE.DELETE, requiredMappingUpdate, id); - this.found = false; - } - public boolean isFound() { return found; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 54223e1e692f3..fe6b0b2051dc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -514,7 +514,11 @@ private static void parseObjectDynamic(DocumentParserContext context, String cur } if (context.dynamic() != ObjectMapper.Dynamic.RUNTIME) { - context.addDynamicMapper(dynamicObjectMapper); + if (context.addDynamicMapper(dynamicObjectMapper) == false) { + failIfMatchesRoutingPath(context, currentFieldName); + context.parser().skipChildren(); + return; + } } if (dynamicObjectMapper instanceof NestedObjectMapper && context.isWithinCopyTo()) { throwOnCreateDynamicNestedViaCopyTo(dynamicObjectMapper, context); @@ -556,7 +560,10 @@ private static void parseArrayDynamic(DocumentParserContext context, String curr parseNonDynamicArray(context, currentFieldName, currentFieldName); } else { if (parsesArrayValue(objectMapperFromTemplate)) { - context.addDynamicMapper(objectMapperFromTemplate); + if (context.addDynamicMapper(objectMapperFromTemplate) == false) { + context.parser().skipChildren(); + return; + } context.path().add(currentFieldName); parseObjectOrField(context, objectMapperFromTemplate); context.path().remove(); @@ -674,7 +681,9 @@ private static void parseDynamicValue(final DocumentParserContext context, Strin failIfMatchesRoutingPath(context, currentFieldName); return; } - context.dynamic().getDynamicFieldsBuilder().createDynamicFieldFromValue(context, currentFieldName); + if (context.dynamic().getDynamicFieldsBuilder().createDynamicFieldFromValue(context, currentFieldName) == false) { + failIfMatchesRoutingPath(context, currentFieldName); + } } private static void ensureNotStrict(DocumentParserContext context, String currentFieldName) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index c0d15da49225e..fc4e290f687b7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -80,14 +80,32 @@ protected void addDoc(LuceneDocument doc) { } } + /** + * Tracks the number of dynamically added mappers. + * All {@link DocumentParserContext}s that are created via {@link DocumentParserContext#createChildContext(ObjectMapper)} + * share the same mutable instance so that we can track the total size of dynamic mappers + * that are added on any level of the object graph. + */ + private static final class DynamicMapperSize { + private int dynamicMapperSize = 0; + + public void add(int mapperSize) { + dynamicMapperSize += mapperSize; + } + + public int get() { + return dynamicMapperSize; + } + } + private final MappingLookup mappingLookup; private final MappingParserContext mappingParserContext; private final SourceToParse sourceToParse; private final Set ignoredFields; private final Map> dynamicMappers; - private final Set newFieldsSeen; + private final DynamicMapperSize dynamicMappersSize; private final Map dynamicObjectMappers; - private final List dynamicRuntimeFields; + private final Map> dynamicRuntimeFields; private final DocumentDimensions dimensions; private final ObjectMapper parent; private final ObjectMapper.Dynamic dynamic; @@ -103,9 +121,8 @@ private DocumentParserContext( SourceToParse sourceToParse, Set ignoreFields, Map> dynamicMappers, - Set newFieldsSeen, Map dynamicObjectMappers, - List dynamicRuntimeFields, + Map> dynamicRuntimeFields, String id, Field version, SeqNoFieldMapper.SequenceIDFields seqID, @@ -113,14 +130,14 @@ private DocumentParserContext( ObjectMapper parent, ObjectMapper.Dynamic dynamic, Set fieldsAppliedFromTemplates, - Set copyToFields + Set copyToFields, + DynamicMapperSize dynamicMapperSize ) { this.mappingLookup = mappingLookup; this.mappingParserContext = mappingParserContext; this.sourceToParse = sourceToParse; this.ignoredFields = ignoreFields; this.dynamicMappers = dynamicMappers; - this.newFieldsSeen = newFieldsSeen; this.dynamicObjectMappers = dynamicObjectMappers; this.dynamicRuntimeFields = dynamicRuntimeFields; this.id = id; @@ -131,6 +148,7 @@ private DocumentParserContext( this.dynamic = dynamic; this.fieldsAppliedFromTemplates = fieldsAppliedFromTemplates; this.copyToFields = copyToFields; + this.dynamicMappersSize = dynamicMapperSize; } private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, DocumentParserContext in) { @@ -140,7 +158,6 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, in.sourceToParse, in.ignoredFields, in.dynamicMappers, - in.newFieldsSeen, in.dynamicObjectMappers, in.dynamicRuntimeFields, in.id, @@ -150,7 +167,8 @@ private DocumentParserContext(ObjectMapper parent, ObjectMapper.Dynamic dynamic, parent, dynamic, in.fieldsAppliedFromTemplates, - in.copyToFields + in.copyToFields, + in.dynamicMappersSize ); } @@ -167,9 +185,8 @@ protected DocumentParserContext( source, new HashSet<>(), new HashMap<>(), - new HashSet<>(), new HashMap<>(), - new ArrayList<>(), + new HashMap<>(), null, null, null, @@ -177,7 +194,8 @@ protected DocumentParserContext( parent, dynamic, new HashSet<>(), - new HashSet<>() + new HashSet<>(), + new DynamicMapperSize() ); } @@ -303,8 +321,13 @@ public boolean isCopyToField(String name) { /** * Add a new mapper dynamically created while parsing. + * + * @return returns true if the mapper could be created, false if the dynamic mapper has been ignored due to + * the field limit + * @throws IllegalArgumentException if the field limit has been exceeded. + * This can happen when dynamic is set to {@link ObjectMapper.Dynamic#TRUE} or {@link ObjectMapper.Dynamic#RUNTIME}. */ - public final void addDynamicMapper(Mapper mapper) { + public final boolean addDynamicMapper(Mapper mapper) { // eagerly check object depth limit here to avoid stack overflow errors if (mapper instanceof ObjectMapper) { MappingLookup.checkObjectDepthLimit(indexSettings().getMappingDepthLimit(), mapper.name()); @@ -315,8 +338,18 @@ public final void addDynamicMapper(Mapper mapper) { // note that existing fields can also receive dynamic mapping updates (e.g. constant_keyword to fix the value) if (mappingLookup.getMapper(mapper.name()) == null && mappingLookup.objectMappers().containsKey(mapper.name()) == false - && newFieldsSeen.add(mapper.name())) { - mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), newFieldsSeen.size()); + && dynamicMappers.containsKey(mapper.name()) == false) { + int mapperSize = mapper.mapperSize(); + int additionalFieldsToAdd = getNewFieldsSize() + mapperSize; + if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { + if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd)) { + addIgnoredField(mapper.name()); + return false; + } + } else { + mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), additionalFieldsToAdd); + } + dynamicMappersSize.add(mapperSize); } if (mapper instanceof ObjectMapper objectMapper) { dynamicObjectMappers.put(objectMapper.name(), objectMapper); @@ -337,6 +370,25 @@ public final void addDynamicMapper(Mapper mapper) { // 1) by default, they would be empty containers in the mappings, is it then important to map them? // 2) they can be the result of applying a dynamic template which may define sub-fields or set dynamic, enabled or subobjects. dynamicMappers.computeIfAbsent(mapper.name(), k -> new ArrayList<>()).add(mapper); + return true; + } + + /* + * Returns an approximation of the number of dynamically mapped fields and runtime fields that will be added to the mapping. + * This is to validate early and to fail fast during document parsing. + * There will be another authoritative (but more expensive) validation step when making the actual update mapping request. + * During the mapping update, the actual number fields is determined by counting the total number of fields of the merged mapping. + * Therefore, both over-counting and under-counting here is not critical. + * However, in order for users to get to the field limit, we should try to be as close as possible to the actual field count. + * If we under-count fields here, we may only know that we exceed the field limit during the mapping update. + * This can happen when merging the mappers for the same field results in a mapper with a larger size than the individual mappers. + * This leads to document rejection instead of ignoring fields above the limit + * if ignore_dynamic_beyond_limit is configured for the index. + * If we over-count the fields (for example by counting all mappers with the same name), + * we may reject fields earlier than necessary and before actually hitting the field limit. + */ + int getNewFieldsSize() { + return dynamicMappersSize.get() + dynamicRuntimeFields.size(); } /** @@ -395,11 +447,19 @@ final ObjectMapper getDynamicObjectMapper(String name) { * because for dynamic mappings, a new field can be either mapped * as runtime or indexed, but never both. */ - final void addDynamicRuntimeField(RuntimeField runtimeField) { - if (newFieldsSeen.add(runtimeField.name())) { - mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), newFieldsSeen.size()); + final boolean addDynamicRuntimeField(RuntimeField runtimeField) { + if (dynamicRuntimeFields.containsKey(runtimeField.name()) == false) { + if (indexSettings().isIgnoreDynamicFieldsBeyondLimit()) { + if (mappingLookup.exceedsLimit(indexSettings().getMappingTotalFieldsLimit(), getNewFieldsSize() + 1)) { + addIgnoredField(runtimeField.name()); + return false; + } + } else { + mappingLookup.checkFieldLimit(indexSettings().getMappingTotalFieldsLimit(), getNewFieldsSize() + 1); + } } - dynamicRuntimeFields.add(runtimeField); + dynamicRuntimeFields.computeIfAbsent(runtimeField.name(), k -> new ArrayList<>(1)).add(runtimeField); + return true; } /** @@ -408,7 +468,7 @@ final void addDynamicRuntimeField(RuntimeField runtimeField) { * or when dynamic templates specify a runtime section. */ public final List getDynamicRuntimeFields() { - return Collections.unmodifiableList(dynamicRuntimeFields); + return dynamicRuntimeFields.values().stream().flatMap(List::stream).toList(); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java index 813fb1a051000..8505c561bfb1a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java @@ -10,9 +10,9 @@ import org.elasticsearch.ElasticsearchParseException; import org.elasticsearch.common.CheckedBiConsumer; +import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.core.CheckedRunnable; import org.elasticsearch.index.mapper.ObjectMapper.Dynamic; import org.elasticsearch.script.ScriptCompiler; import org.elasticsearch.xcontent.XContentParser; @@ -44,7 +44,7 @@ private DynamicFieldsBuilder(Strategy strategy) { * delegates to the appropriate strategy which depends on the current dynamic mode. * The strategy defines if fields are going to be mapped as ordinary or runtime fields. */ - void createDynamicFieldFromValue(final DocumentParserContext context, String name) throws IOException { + boolean createDynamicFieldFromValue(final DocumentParserContext context, String name) throws IOException { XContentParser.Token token = context.parser().currentToken(); if (token == XContentParser.Token.VALUE_STRING) { String text = context.parser().text(); @@ -66,14 +66,14 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam } if (parseableAsLong && context.root().numericDetection()) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.LONG, () -> strategy.newDynamicLongField(context, name) ); } else if (parseableAsDouble && context.root().numericDetection()) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.DOUBLE, @@ -90,22 +90,21 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam // failure to parse this, continue continue; } - createDynamicDateField( + return createDynamicDateField( context, name, dateTimeFormatter, () -> strategy.newDynamicDateField(context, name, dateTimeFormatter) ); - return; } - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.STRING, () -> strategy.newDynamicStringField(context, name) ); } else { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.STRING, @@ -117,7 +116,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG || numberType == XContentParser.NumberType.BIG_INTEGER) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.LONG, @@ -126,7 +125,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE || numberType == XContentParser.NumberType.BIG_DECIMAL) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.DOUBLE, @@ -136,7 +135,7 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam throw new IllegalStateException("Unable to parse number of type [" + numberType + "]"); } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.BOOLEAN, @@ -144,14 +143,14 @@ void createDynamicFieldFromValue(final DocumentParserContext context, String nam ); } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { // runtime binary fields are not supported, hence binary objects always get created as concrete fields - createDynamicField( + return createDynamicField( context, name, DynamicTemplate.XContentFieldType.BINARY, () -> CONCRETE.newDynamicBinaryField(context, name) ); } else { - createDynamicStringFieldFromTemplate(context, name); + return createDynamicStringFieldFromTemplate(context, name); } } @@ -178,40 +177,41 @@ static Mapper createObjectMapperFromTemplate(DocumentParserContext context, Stri * Creates a dynamic string field based on a matching dynamic template. * No field is created in case there is no matching dynamic template. */ - static void createDynamicStringFieldFromTemplate(DocumentParserContext context, String name) throws IOException { - createDynamicField(context, name, DynamicTemplate.XContentFieldType.STRING, () -> {}); + static boolean createDynamicStringFieldFromTemplate(DocumentParserContext context, String name) throws IOException { + return createDynamicField(context, name, DynamicTemplate.XContentFieldType.STRING, () -> false); } - private static void createDynamicDateField( + private static boolean createDynamicDateField( DocumentParserContext context, String name, DateFormatter dateFormatter, - CheckedRunnable createDynamicField + CheckedSupplier createDynamicField ) throws IOException { - createDynamicField(context, name, DynamicTemplate.XContentFieldType.DATE, dateFormatter, createDynamicField); + return createDynamicField(context, name, DynamicTemplate.XContentFieldType.DATE, dateFormatter, createDynamicField); } - private static void createDynamicField( + private static boolean createDynamicField( DocumentParserContext context, String name, DynamicTemplate.XContentFieldType matchType, - CheckedRunnable dynamicFieldStrategy + CheckedSupplier dynamicFieldStrategy ) throws IOException { assert matchType != DynamicTemplate.XContentFieldType.DATE; - createDynamicField(context, name, matchType, null, dynamicFieldStrategy); + return createDynamicField(context, name, matchType, null, dynamicFieldStrategy); } - private static void createDynamicField( + private static boolean createDynamicField( DocumentParserContext context, String name, DynamicTemplate.XContentFieldType matchType, DateFormatter dateFormatter, - CheckedRunnable dynamicFieldStrategy + CheckedSupplier dynamicFieldStrategy ) throws IOException { if (applyMatchingTemplate(context, name, matchType, dateFormatter)) { context.markFieldAsAppliedFromTemplate(name); + return true; } else { - dynamicFieldStrategy.run(); + return dynamicFieldStrategy.get(); } } @@ -284,15 +284,15 @@ private static Mapper.Builder parseDynamicTemplateMapping( * Defines how leaf fields of type string, long, double, boolean and date are dynamically mapped */ private interface Strategy { - void newDynamicStringField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException; - void newDynamicLongField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException; - void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException; - void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException; + boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException; - void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException; + boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) throws IOException; } /** @@ -307,28 +307,32 @@ private static final class Concrete implements Strategy { this.parseField = parseField; } - void createDynamicField(Mapper.Builder builder, DocumentParserContext context, MapperBuilderContext mapperBuilderContext) + boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context, MapperBuilderContext mapperBuilderContext) throws IOException { Mapper mapper = builder.build(mapperBuilderContext); - context.addDynamicMapper(mapper); - parseField.accept(context, mapper); + if (context.addDynamicMapper(mapper)) { + parseField.accept(context, mapper); + return true; + } else { + return false; + } } - void createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { - createDynamicField(builder, context, context.createDynamicMapperBuilderContext()); + boolean createDynamicField(Mapper.Builder builder, DocumentParserContext context) throws IOException { + return createDynamicField(builder, context, context.createDynamicMapperBuilderContext()); } @Override - public void newDynamicStringField(DocumentParserContext context, String name) throws IOException { + public boolean newDynamicStringField(DocumentParserContext context, String name) throws IOException { MapperBuilderContext mapperBuilderContext = context.createDynamicMapperBuilderContext(); if (mapperBuilderContext.parentObjectContainsDimensions()) { - createDynamicField( + return createDynamicField( new KeywordFieldMapper.Builder(name, context.indexSettings().getIndexVersionCreated()), context, mapperBuilderContext ); } else { - createDynamicField( + return createDynamicField( new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField( new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256) ), @@ -338,8 +342,8 @@ public void newDynamicStringField(DocumentParserContext context, String name) th } @Override - public void newDynamicLongField(DocumentParserContext context, String name) throws IOException { - createDynamicField( + public boolean newDynamicLongField(DocumentParserContext context, String name) throws IOException { + return createDynamicField( new NumberFieldMapper.Builder( name, NumberFieldMapper.NumberType.LONG, @@ -353,11 +357,11 @@ public void newDynamicLongField(DocumentParserContext context, String name) thro } @Override - public void newDynamicDoubleField(DocumentParserContext context, String name) throws IOException { + public boolean newDynamicDoubleField(DocumentParserContext context, String name) throws IOException { // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of // the time - createDynamicField( + return createDynamicField( new NumberFieldMapper.Builder( name, NumberFieldMapper.NumberType.FLOAT, @@ -371,10 +375,10 @@ public void newDynamicDoubleField(DocumentParserContext context, String name) th } @Override - public void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException { + public boolean newDynamicBooleanField(DocumentParserContext context, String name) throws IOException { Settings settings = context.indexSettings().getSettings(); boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings); - createDynamicField( + return createDynamicField( new BooleanFieldMapper.Builder( name, ScriptCompiler.NONE, @@ -386,10 +390,10 @@ public void newDynamicBooleanField(DocumentParserContext context, String name) t } @Override - public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateTimeFormatter) throws IOException { + public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateTimeFormatter) throws IOException { Settings settings = context.indexSettings().getSettings(); boolean ignoreMalformed = FieldMapper.IGNORE_MALFORMED_SETTING.get(settings); - createDynamicField( + return createDynamicField( new DateFieldMapper.Builder( name, DateFieldMapper.Resolution.MILLISECONDS, @@ -402,8 +406,8 @@ public void newDynamicDateField(DocumentParserContext context, String name, Date ); } - void newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { - createDynamicField(new BinaryFieldMapper.Builder(name), context); + boolean newDynamicBinaryField(DocumentParserContext context, String name) throws IOException { + return createDynamicField(new BinaryFieldMapper.Builder(name), context); } } @@ -413,40 +417,43 @@ void newDynamicBinaryField(DocumentParserContext context, String name) throws IO * @see Dynamic */ private static final class Runtime implements Strategy { - static void createDynamicField(RuntimeField runtimeField, DocumentParserContext context) { - context.addDynamicRuntimeField(runtimeField); + static boolean createDynamicField(RuntimeField runtimeField, DocumentParserContext context) { + return context.addDynamicRuntimeField(runtimeField); } @Override - public void newDynamicStringField(DocumentParserContext context, String name) { + public boolean newDynamicStringField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(KeywordScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(KeywordScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicLongField(DocumentParserContext context, String name) { + public boolean newDynamicLongField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(LongScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(LongScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicDoubleField(DocumentParserContext context, String name) { + public boolean newDynamicDoubleField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(DoubleScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(DoubleScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicBooleanField(DocumentParserContext context, String name) { + public boolean newDynamicBooleanField(DocumentParserContext context, String name) { String fullName = context.path().pathAsText(name); - createDynamicField(BooleanScriptFieldType.sourceOnly(fullName), context); + return createDynamicField(BooleanScriptFieldType.sourceOnly(fullName), context); } @Override - public void newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) { + public boolean newDynamicDateField(DocumentParserContext context, String name, DateFormatter dateFormatter) { String fullName = context.path().pathAsText(name); MappingParserContext parserContext = context.dynamicTemplateParserContext(dateFormatter); - createDynamicField(DateScriptFieldType.sourceOnly(fullName, dateFormatter, parserContext.indexVersionCreated()), context); + return createDynamicField( + DateScriptFieldType.sourceOnly(fullName, dateFormatter, parserContext.indexVersionCreated()), + context + ); } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java index 79adaf5966c5b..0af182f315559 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java @@ -96,7 +96,6 @@ private Unlimited() {} public boolean decrementIfPossible(long fieldSize) { return true; } - } final class Limited implements NewFieldsBudget { @@ -115,7 +114,6 @@ public boolean decrementIfPossible(long fieldSize) { } return false; } - } } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index bde9df167efb4..9e47b42c4b18e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.function.Function; import java.util.function.Supplier; @@ -118,6 +119,12 @@ public boolean isAutoUpdate() { Property.IndexScope, Property.ServerlessPublic ); + public static final Setting INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING = Setting.boolSetting( + "index.mapping.total_fields.ignore_dynamic_beyond_limit", + false, + Property.Dynamic, + Property.IndexScope + ); public static final Setting INDEX_MAPPING_DEPTH_LIMIT_SETTING = Setting.longSetting( "index.mapping.depth.limit", 20L, @@ -331,7 +338,7 @@ public void updateMapping(final IndexMetadata currentIndexMetadata, final IndexM } private boolean assertRefreshIsNotNeeded(DocumentMapper currentMapper, String type, Mapping incomingMapping) { - Mapping mergedMapping = mergeMappings(currentMapper, incomingMapping, MergeReason.MAPPING_RECOVERY); + Mapping mergedMapping = mergeMappings(currentMapper, incomingMapping, MergeReason.MAPPING_RECOVERY, indexSettings); // skip the runtime section or removed runtime fields will make the assertion fail ToXContent.MapParams params = new ToXContent.MapParams(Collections.singletonMap(RootObjectMapper.TOXCONTENT_SKIP_RUNTIME, "true")); CompressedXContent mergedMappingSource; @@ -535,7 +542,7 @@ public DocumentMapper merge(String type, CompressedXContent mappingSource, Merge private synchronized DocumentMapper doMerge(String type, MergeReason reason, Map mappingSourceAsMap) { Mapping incomingMapping = parseMapping(type, mappingSourceAsMap); - Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason); + Mapping mapping = mergeMappings(this.mapper, incomingMapping, reason, this.indexSettings); // TODO: In many cases the source here is equal to mappingSource so we need not serialize again. // We should identify these cases reliably and save expensive serialization here DocumentMapper newMapper = newDocumentMapper(mapping, reason, mapping.toCompressedXContent()); @@ -576,8 +583,38 @@ public Mapping parseMapping(String mappingType, Map mappingSourc } } - public static Mapping mergeMappings(DocumentMapper currentMapper, Mapping incomingMapping, MergeReason reason) { - return mergeMappings(currentMapper, incomingMapping, reason, Long.MAX_VALUE); + public static Mapping mergeMappings( + DocumentMapper currentMapper, + Mapping incomingMapping, + MergeReason reason, + IndexSettings indexSettings + ) { + return mergeMappings(currentMapper, incomingMapping, reason, getMaxFieldsToAddDuringMerge(currentMapper, indexSettings, reason)); + } + + private static long getMaxFieldsToAddDuringMerge(DocumentMapper currentMapper, IndexSettings indexSettings, MergeReason reason) { + if (reason.isAutoUpdate() && indexSettings.isIgnoreDynamicFieldsBeyondLimit()) { + // If the index setting ignore_dynamic_beyond_limit is enabled, + // data nodes only add new dynamic fields until the limit is reached while parsing documents to be ingested. + // However, if there are concurrent mapping updates, + // data nodes may add dynamic fields under an outdated assumption that enough capacity is still available. + // When data nodes send the dynamic mapping update request to the master node, + // it will only add as many fields as there's actually capacity for when merging mappings. + long totalFieldsLimit = indexSettings.getMappingTotalFieldsLimit(); + return Optional.ofNullable(currentMapper) + .map(DocumentMapper::mappers) + .map(ml -> ml.remainingFieldsUntilLimit(totalFieldsLimit)) + .orElse(totalFieldsLimit); + } else { + // Else, we're not limiting the number of fields so that the merged mapping fails validation if it exceeds total_fields.limit. + // This is the desired behavior when making an explicit mapping update, even if ignore_dynamic_beyond_limit is enabled. + // When ignore_dynamic_beyond_limit is disabled and a dynamic mapping update would exceed the field limit, + // the document will get rejected. + // Normally, this happens on the data node in DocumentParserContext.addDynamicMapper but if there's a race condition, + // data nodes may add dynamic fields under an outdated assumption that enough capacity is still available. + // In this case, the master node will reject mapping updates that would exceed the limit when handling the mapping update. + return Long.MAX_VALUE; + } } static Mapping mergeMappings(DocumentMapper currentMapper, Mapping incomingMapping, MergeReason reason, long newFieldsBudget) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 0172c22c0b176..ea59d6640f647 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -282,7 +282,11 @@ void checkFieldLimit(long limit, int additionalFieldsToAdd) { } boolean exceedsLimit(long limit, int additionalFieldsToAdd) { - return getTotalFieldsCount() + additionalFieldsToAdd - mapping.getSortedMetadataMappers().length > limit; + return remainingFieldsUntilLimit(limit) < additionalFieldsToAdd; + } + + long remainingFieldsUntilLimit(long mappingTotalFieldsLimit) { + return mappingTotalFieldsLimit - getTotalFieldsCount() + mapping.getSortedMetadataMappers().length; } private void checkDimensionFieldLimit(long limit) { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java index bf540eae5ed49..0c604cb171457 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ParsedDocument.java @@ -35,7 +35,6 @@ public class ParsedDocument { private BytesReference source; private XContentType xContentType; - private Mapping dynamicMappingsUpdate; /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java index 8c4fae79c797c..86bdb2aa2bba7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/RootObjectMapper.java @@ -308,10 +308,8 @@ RootObjectMapper merge(RootObjectMapper mergeWithObject, MergeReason reason, Map runtimeFields.remove(runtimeField.getKey()); } else if (runtimeFields.containsKey(runtimeField.getKey())) { runtimeFields.put(runtimeField.getKey(), runtimeField.getValue()); - } else { - if (parentMergeContext.decrementFieldBudgetIfPossible(1)) { - runtimeFields.put(runtimeField.getValue().name(), runtimeField.getValue()); - } + } else if (parentMergeContext.decrementFieldBudgetIfPossible(1)) { + runtimeFields.put(runtimeField.getValue().name(), runtimeField.getValue()); } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java new file mode 100644 index 0000000000000..03716f8ad4497 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserContextTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.contains; + +public class DocumentParserContextTests extends ESTestCase { + + private TestDocumentParserContext context = new TestDocumentParserContext(); + private final MapperBuilderContext root = MapperBuilderContext.root(false, false); + + public void testDynamicMapperSizeMultipleMappers() { + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + context.addDynamicMapper(new TextFieldMapper.Builder("bar", createDefaultIndexAnalyzers()).build(root)); + assertEquals(2, context.getNewFieldsSize()); + context.addDynamicRuntimeField(new TestRuntimeField("runtime1", "keyword")); + assertEquals(3, context.getNewFieldsSize()); + context.addDynamicRuntimeField(new TestRuntimeField("runtime2", "keyword")); + assertEquals(4, context.getNewFieldsSize()); + } + + public void testDynamicMapperSizeSameFieldMultipleRuntimeFields() { + context.addDynamicRuntimeField(new TestRuntimeField("foo", "keyword")); + context.addDynamicRuntimeField(new TestRuntimeField("foo", "keyword")); + assertEquals(context.getNewFieldsSize(), 1); + } + + public void testDynamicMapperSizeSameFieldMultipleMappers() { + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + context.addDynamicMapper(new TextFieldMapper.Builder("foo", createDefaultIndexAnalyzers()).build(root)); + assertEquals(1, context.getNewFieldsSize()); + } + + public void testAddRuntimeFieldWhenLimitIsReachedViaMapper() { + context = new TestDocumentParserContext( + Settings.builder() + .put("index.mapping.total_fields.limit", 1) + .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) + .build() + ); + assertTrue(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertFalse(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); + assertThat(context.getIgnoredFields(), contains("runtime_field")); + } + + public void testAddFieldWhenLimitIsReachedViaRuntimeField() { + context = new TestDocumentParserContext( + Settings.builder() + .put("index.mapping.total_fields.limit", 1) + .put("index.mapping.total_fields.ignore_dynamic_beyond_limit", true) + .build() + ); + assertTrue(context.addDynamicRuntimeField(new TestRuntimeField("runtime_field", "keyword"))); + assertFalse(context.addDynamicMapper(new KeywordFieldMapper.Builder("keyword_field", IndexVersion.current()).build(root))); + assertThat(context.getIgnoredFields(), contains("keyword_field")); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java index a3706b7ddab18..ed2efb4728b8d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DocumentParserTests.java @@ -2580,7 +2580,12 @@ same name need to be part of the same mappings (hence the same document). If th assertArrayEquals(new String[] { "LongField ", "LongField " }, fieldStrings); // merge without going through toXContent and reparsing, otherwise the potential leaf path issue gets fixed on its own - Mapping newMapping = MapperService.mergeMappings(mapperService.documentMapper(), mapping, MapperService.MergeReason.MAPPING_UPDATE); + Mapping newMapping = MapperService.mergeMappings( + mapperService.documentMapper(), + mapping, + MapperService.MergeReason.MAPPING_UPDATE, + mapperService.getIndexSettings() + ); DocumentMapper newDocMapper = new DocumentMapper( mapperService.documentParser(), newMapping, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java index 80c074918b06d..68e7bd6f24664 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MapperServiceTests.java @@ -24,9 +24,14 @@ import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.StreamSupport; +import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; @@ -66,6 +71,7 @@ public void testTotalFieldsLimit() throws Throwable { int totalFieldsLimit = randomIntBetween(1, 10); Settings settings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), totalFieldsLimit) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); MapperService mapperService = createMapperService( settings, @@ -173,6 +179,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { Settings settings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfFieldsIncludingAlias) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); createMapperService(settings, mapping(b -> { b.startObject("alias").field("type", "alias").field("path", "field").endObject(); @@ -184,6 +191,7 @@ public void testTotalFieldsLimitWithFieldAlias() throws Throwable { int numberOfNonAliasFields = 1; Settings errorSettings = Settings.builder() .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), numberOfNonAliasFields) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) .build(); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> createMapperService(errorSettings, mapping(b -> { b.startObject("alias").field("type", "alias").field("path", "field").endObject(); @@ -1241,4 +1249,222 @@ public void testPropertiesField() throws IOException { assertThat(grandchildMapper, instanceOf(FieldMapper.class)); assertEquals("keyword", grandchildMapper.typeName()); } + + public void testMergeUntilLimit() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "keyword" + } + } + }"""); + + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "parent.child2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("parent.child1")); + assertNull(mapper.mappers().getMapper("parent.child2")); + } + + public void testMergeUntilLimitMixedObjectAndDottedNotation() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "parent": { + "properties": { + "child1": { + "type": "keyword" + } + } + }, + "parent.child2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertEquals(0, mapper.mappers().remainingFieldsUntilLimit(2)); + assertNotNull(mapper.mappers().objectMappers().get("parent")); + // the order is not deterministic, but we expect one to be null and the other to be non-null + assertTrue(mapper.mappers().getMapper("parent.child1") == null ^ mapper.mappers().getMapper("parent.child2") == null); + } + + public void testUpdateMappingWhenAtLimit() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "boolean" + } + } + }"""); + + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "parent.child1": { + "type": "boolean", + "ignore_malformed": true + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("parent.child1")); + assertTrue(((BooleanFieldMapper) mapper.mappers().getMapper("parent.child1")).ignoreMalformed()); + } + + public void testMultiFieldsUpdate() throws IOException { + CompressedXContent mapping1 = new CompressedXContent(""" + { + "properties": { + "text_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean" + } + } + } + } + }"""); + + // changes a mapping parameter for multi_field1 and adds another multi field which is supposed to be ignored + CompressedXContent mapping2 = new CompressedXContent(""" + { + "properties": { + "text_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean", + "ignore_malformed": true + }, + "multi_field2": { + "type": "keyword" + } + } + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 2) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping1, MergeReason.MAPPING_AUTO_UPDATE); + mapper = mapperService.merge("_doc", mapping2, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().getMapper("text_field")); + FieldMapper.MultiFields multiFields = ((TextFieldMapper) mapper.mappers().getMapper("text_field")).multiFields(); + Map multiFieldMap = StreamSupport.stream(multiFields.spliterator(), false) + .collect(Collectors.toMap(FieldMapper::name, Function.identity())); + assertThat(multiFieldMap.keySet(), contains("text_field.multi_field1")); + assertTrue(multiFieldMap.get("text_field.multi_field1").ignoreMalformed()); + } + + public void testMultiFieldExceedsLimit() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "multi_field": { + "type": "text", + "fields": { + "multi_field1": { + "type": "boolean" + } + } + }, + "keyword_field": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertNull(mapper.mappers().getMapper("multi_field")); + assertNotNull(mapper.mappers().getMapper("keyword_field")); + } + + public void testMergeUntilLimitInitialMappingExceedsLimit() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "field1": { + "type": "keyword" + }, + "field2": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + // the order is not deterministic, but we expect one to be null and the other to be non-null + assertTrue(mapper.mappers().getMapper("field1") == null ^ mapper.mappers().getMapper("field2") == null); + } + + public void testMergeUntilLimitCapacityOnlyForParent() throws IOException { + CompressedXContent mapping = new CompressedXContent(""" + { + "properties": { + "parent.child": { + "type": "keyword" + } + } + }"""); + + Settings settings = Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1) + .put(INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING.getKey(), true) + .build(); + + final MapperService mapperService = createMapperService(settings, mapping(b -> {})); + DocumentMapper mapper = mapperService.merge("_doc", mapping, MergeReason.MAPPING_AUTO_UPDATE); + assertNotNull(mapper.mappers().objectMappers().get("parent")); + assertNull(mapper.mappers().getMapper("parent.child")); + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java index 4e953d02e4d81..d4c238322e28a 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/TestDocumentParserContext.java @@ -34,8 +34,12 @@ public TestDocumentParserContext() { this(MappingLookup.EMPTY, null); } + public TestDocumentParserContext(Settings settings) { + this(MappingLookup.EMPTY, null, null, settings); + } + public TestDocumentParserContext(XContentParser parser) { - this(MappingLookup.EMPTY, null, parser); + this(MappingLookup.EMPTY, null, parser, Settings.EMPTY); } /** @@ -43,10 +47,10 @@ public TestDocumentParserContext(XContentParser parser) { * that depend on them are called while executing tests. */ public TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source) { - this(mappingLookup, source, null); + this(mappingLookup, source, null, Settings.EMPTY); } - private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source, XContentParser parser) { + private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse source, XContentParser parser, Settings settings) { super( mappingLookup, new MappingParserContext( @@ -58,7 +62,7 @@ private TestDocumentParserContext(MappingLookup mappingLookup, SourceToParse sou () -> null, null, (type, name) -> Lucene.STANDARD_ANALYZER, - MapperTestCase.createIndexSettings(IndexVersion.current(), Settings.EMPTY), + MapperTestCase.createIndexSettings(IndexVersion.current(), settings), null ), source, diff --git a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java index 086ab7d842eb1..1313e5781f122 100644 --- a/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java +++ b/x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/action/TransportResumeFollowActionTests.java @@ -319,6 +319,7 @@ public void testDynamicIndexSettingsAreClassified() { // These fields need to be replicated otherwise documents that can be indexed in the leader index cannot // be indexed in the follower index: replicatedSettings.add(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING); + replicatedSettings.add(MapperService.INDEX_MAPPING_IGNORE_DYNAMIC_BEYOND_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING); replicatedSettings.add(MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index cc819c353f69c..cee397d906149 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -311,7 +311,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio if (fieldType().value == null) { ConstantKeywordFieldType newFieldType = new ConstantKeywordFieldType(fieldType().name(), value, fieldType().meta()); Mapper update = new ConstantKeywordFieldMapper(simpleName(), newFieldType); - context.addDynamicMapper(update); + boolean dynamicMapperAdded = context.addDynamicMapper(update); + // the mapper is already part of the mapping, we're just updating it with the new value + assert dynamicMapperAdded; } else if (Objects.equals(fieldType().value, value) == false) { throw new IllegalArgumentException( "[constant_keyword] field ["