From 682b0d529d9e225fff92f151c8a834af36387a8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Mon, 29 Oct 2018 09:34:46 +0100 Subject: [PATCH] Add warning about upcoming expanded fields limit (#34906) Starting with Elasticsearch 7.0, we limit the number of automatically expanded fields for the "all fields" mode ("default_field": "*") for the query_string and simple_query_string queries to 1024 fields (see #26541). This change adds a deprecation warning to the QueryParserHelper where in the next major version we will throw an error to warn users of the upcoming change. Relates to #26541 --- docs/reference/migration/index.asciidoc | 2 + docs/reference/migration/migrate_6_6.asciidoc | 24 +++++++ .../query-dsl/query-string-query.asciidoc | 4 ++ .../simple-query-string-query.asciidoc | 4 ++ .../index/search/QueryParserHelper.java | 16 +++++ .../index/search/QueryParserHelperTests.java | 67 +++++++++++++++++++ 6 files changed, 117 insertions(+) create mode 100644 docs/reference/migration/migrate_6_6.asciidoc create mode 100644 server/src/test/java/org/elasticsearch/index/search/QueryParserHelperTests.java diff --git a/docs/reference/migration/index.asciidoc b/docs/reference/migration/index.asciidoc index 31bb9414af25d..4494dc2709926 100644 --- a/docs/reference/migration/index.asciidoc +++ b/docs/reference/migration/index.asciidoc @@ -33,3 +33,5 @@ include::migrate_6_3.asciidoc[] include::migrate_6_4.asciidoc[] include::migrate_6_5.asciidoc[] + +include::migrate_6_6.asciidoc[] diff --git a/docs/reference/migration/migrate_6_6.asciidoc b/docs/reference/migration/migrate_6_6.asciidoc new file mode 100644 index 0000000000000..e19fad8de0fba --- /dev/null +++ b/docs/reference/migration/migrate_6_6.asciidoc @@ -0,0 +1,24 @@ +[[breaking-changes-6.6]] +== Breaking changes in 6.6 +++++ +6.6 +++++ + +This section discusses the changes that you need to be aware of when migrating +your application to Elasticsearch 6.6. + +* <> + +See also <> and <>. + +[float] +[[breaking_66_search_changes]] +=== Search changes + +[float] +==== `query_string`, `multi_match` and `simple_query_string` query + +Using automatically expanded fields for the "all fields" mode ("default_field": "*") +for the `query_string`, `multi_match` and `simple_query_string` now raises a warning and +a deprecation notice to be logged for queries beyond 1024 fields. This limit will be +enforced with a hard error starting in 7.0. diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc index fedc5bd0ab65c..e2c969348c9ed 100644 --- a/docs/reference/query-dsl/query-string-query.asciidoc +++ b/docs/reference/query-dsl/query-string-query.asciidoc @@ -63,6 +63,10 @@ settings, which in turn defaults to `*`. and filters the metadata fields. All extracted fields are then combined to build a query when no prefix field is provided. +WARNING The amount of fields being allowed to be queries at once will be limited +to 1024 in the next major version or Elasticsearch. Currently this will be raised +and logged as a Warning only. + |`default_operator` |The default operator used if no explicit operator is specified. For example, with a default operator of `OR`, the query `capital of Hungary` is translated to `capital OR of OR Hungary`, and diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc index bec4ea365c15c..14e948f28cf38 100644 --- a/docs/reference/query-dsl/simple-query-string-query.asciidoc +++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc @@ -33,6 +33,10 @@ The `simple_query_string` top level parameters include: `*` extracts all fields in the mapping that are eligible to term queries and filters the metadata fields. +WARNING The amount of fields being allowed to be queries at once will be limited +to 1024 in the next major version or Elasticsearch. Currently this will be raised +and logged as a Warning only. + |`default_operator` |The default operator used if no explicit operator is specified. For example, with a default operator of `OR`, the query `capital of Hungary` is translated to `capital OR of OR Hungary`, and diff --git a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java index 6fc1ae51bde2d..521ce0ec68d9d 100644 --- a/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java +++ b/server/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.search; +import org.apache.logging.log4j.LogManager; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.regex.Regex; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.QueryShardContext; @@ -34,6 +36,9 @@ * Helpers to extract and expand field names and boosts */ public final class QueryParserHelper { + + private static final DeprecationLogger DEPRECATION_LOGGER = new DeprecationLogger(LogManager.getLogger(QueryParserHelper.class)); + private QueryParserHelper() {} /** @@ -85,6 +90,7 @@ public static Map resolveMappingFields(QueryShardContext context, !multiField, !allField, fieldSuffix); resolvedFields.putAll(fieldMap); } + checkForTooManyFields(resolvedFields); return resolvedFields; } @@ -149,6 +155,16 @@ public static Map resolveMappingField(QueryShardContext context, } fields.put(fieldName, weight); } + checkForTooManyFields(fields); return fields; } + + private static void checkForTooManyFields(Map fields) { + if (fields.size() > 1024) { + DEPRECATION_LOGGER.deprecatedAndMaybeLog("field_expansion_limit", + "Field expansion matches too many fields, got: {}. A limit of 1024 will be enforced starting " + + "with version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading.", + fields.size()); + } + } } diff --git a/server/src/test/java/org/elasticsearch/index/search/QueryParserHelperTests.java b/server/src/test/java/org/elasticsearch/index/search/QueryParserHelperTests.java new file mode 100644 index 0000000000000..d9be336ce5fac --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/search/QueryParserHelperTests.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.search; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.util.Collections; + +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; + +public class QueryParserHelperTests extends ESSingleNodeTestCase { + + /** + * Test that when {@link QueryParserHelper#resolveMappingFields(QueryShardContext, java.util.Map, String)} exceeds + * the limit of 1024 fields, we emit a warning + */ + public void testLimitOnExpandedFields() throws Exception { + + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject("type1"); + builder.startObject("properties"); + for (int i = 0; i < 1025; i++) { + builder.startObject("field" + i).field("type", "text").endObject(); + } + builder.endObject(); // properties + builder.endObject(); // type1 + builder.endObject(); + IndexService indexService = createIndex("toomanyfields", Settings.builder() + .put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200).build(), + "type1", builder); + client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get(); + + QueryShardContext queryShardContext = indexService.newQueryShardContext( + randomInt(20), null, () -> { throw new UnsupportedOperationException(); }, null); + + QueryParserHelper.resolveMappingField(queryShardContext, "*", 1.0f, true, false); + assertWarnings("Field expansion matches too many fields, got: 1025. A limit of 1024 will be enforced starting with " + + "version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading."); + + QueryParserHelper.resolveMappingFields(queryShardContext,Collections.singletonMap("*", 1.0f)); + assertWarnings("Field expansion matches too many fields, got: 1025. A limit of 1024 will be enforced starting with " + + "version 7.0 of Elasticsearch. Lowering the number of fields will be necessary before upgrading."); + } +}