New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add limit to total number of fields in mapping #17357

Closed
wants to merge 1 commit into
base: master
from
Jump to file or symbol
Failed to load files and symbols.
+43 −1
Diff settings

Always

Just for now

@@ -128,6 +128,7 @@
PercolatorQueryCache.INDEX_MAP_UNMAPPED_FIELDS_AS_STRING_SETTING,
MapperService.INDEX_MAPPER_DYNAMIC_SETTING,
MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING,
MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING,
BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING,
IndexModule.INDEX_STORE_TYPE_SETTING,
IndexModule.INDEX_QUERY_CACHE_TYPE_SETTING,
@@ -84,6 +84,8 @@
public static final String DEFAULT_MAPPING = "_default_";
public static final Setting<Long> INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING =
Setting.longSetting("index.mapping.nested_fields.limit", 50L, 0, Property.Dynamic, Property.IndexScope);
public static final Setting<Long> INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING =
Setting.longSetting("index.mapping.total_fields.limit", 1000L, 0, Property.Dynamic, Property.IndexScope);
public static final boolean INDEX_MAPPER_DYNAMIC_DEFAULT = true;
public static final Setting<Boolean> INDEX_MAPPER_DYNAMIC_SETTING =
Setting.boolSetting("index.mapper.dynamic", INDEX_MAPPER_DYNAMIC_DEFAULT, Property.IndexScope);
@@ -284,6 +286,7 @@ private synchronized DocumentMapper merge(DocumentMapper mapper, MergeReason rea
if (reason == MergeReason.MAPPING_UPDATE) {
checkNestedFieldsLimit(fullPathObjectMappers);
checkTotalFieldsLimit(objectMappers.size() + fieldMappers.size());
}
Set<String> parentTypes = this.parentTypes;
@@ -403,6 +406,13 @@ private void checkNestedFieldsLimit(Map<String, ObjectMapper> fullPathObjectMapp
}
}
private void checkTotalFieldsLimit(long totalMappers) {
long allowedTotalFields = indexSettings.getValue(INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING);
if (allowedTotalFields > 0 && allowedTotalFields < totalMappers) {

This comment has been minimized.

@jpountz

jpountz Mar 29, 2016

Contributor

Like Simon suggested, I would just do: if (allowedTotalFields < totalMappers) {. If someone needs to have many fields anyway and understands the issue, (s)he can still set index.mapping.total_fields.limit=1000000 for instance.

@jpountz

jpountz Mar 29, 2016

Contributor

Like Simon suggested, I would just do: if (allowedTotalFields < totalMappers) {. If someone needs to have many fields anyway and understands the issue, (s)he can still set index.mapping.total_fields.limit=1000000 for instance.

throw new IllegalArgumentException("Limit of total fields [" + allowedTotalFields + "] in index [" + index().getName() + "] has been exceeded");
}
}
public DocumentMapper parse(String mappingType, CompressedXContent mappingSource, boolean applyDefault) throws MapperParsingException {
String defaultMappingSource;
if (PercolatorFieldMapper.TYPE_NAME.equals(mappingType)) {
@@ -31,6 +31,7 @@
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.hamcrest.CollectionAssertions;
import org.junit.Before;
@@ -145,7 +146,7 @@ public void testLargeClusterStatePublishing() throws Exception {
int numberOfShards = scaledRandomIntBetween(1, cluster().numDataNodes());
// if the create index is ack'ed, then all nodes have successfully processed the cluster state
assertAcked(client().admin().indices().prepareCreate("test")
.setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.setSettings(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numberOfShards, IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0, MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 0)
.addMapping("type", mapping)
.setTimeout("60s").get());
ensureGreen(); // wait for green state, so its both green, and there are no more pending events
@@ -30,15 +30,21 @@
import org.elasticsearch.ExceptionsHelper;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.lucene.search.Queries;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.IndexService;
import org.elasticsearch.index.mapper.MapperService.MergeReason;
import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
import org.elasticsearch.test.ESSingleNodeTestCase;
import org.junit.Rule;
import org.junit.rules.ExpectedException;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.function.Function;
import java.util.concurrent.ExecutionException;
import static org.hamcrest.CoreMatchers.containsString;
@@ -135,4 +141,24 @@ public void testIndexIntoDefaultMapping() throws Throwable {
assertFalse(indexService.mapperService().hasMapping(MapperService.DEFAULT_MAPPING));
}
public void testTotalFieldsExceedsLimit() throws Throwable {
Function<String, String> mapping = type -> {
try {
return XContentFactory.jsonBuilder().startObject().startObject(type).startObject("properties")
.startObject("field1").field("type", "string")
.endObject().endObject().endObject().endObject().string();
} catch (IOException e) {
throw new UncheckedIOException(e);
}
};
createIndex("test1").mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
//set total number of fields to 1 to trigger an exception
try {
createIndex("test2", Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1).build())
.mapperService().merge("type", new CompressedXContent(mapping.apply("type")), MergeReason.MAPPING_UPDATE, false);
fail("Expected IllegalArgumentException");
} catch (IllegalArgumentException e) {
assertThat(e.getMessage(), containsString("Limit of total fields [1] in index [test2] has been exceeded"));
}
}
}
@@ -30,6 +30,10 @@ detected. All other datatypes must be mapped explicitly.
Besides the options listed below, dynamic field mapping rules can be further
customised with <<dynamic-templates,`dynamic_templates`>>.
[[total-fields-limit]]
==== Total fields limit
To avoid mapping explosion, Index has a default limit of 1000 total number of fields. The default setting can be updated with `index.mapping.total_fields.limit`. Setting it to value 0 disables this checking.
[[date-detection]]
==== Date detection
ProTip! Use n and p to navigate between commits in a pull request.