Restore use of default search and search_quote analyzers (#65491) (#6…

…5562) In the refactoring of TextFieldMapper, we lost the ability to define a default search or search_quote analyzer in index settings. This commit restores that ability, and adds some more comprehensive testing. Fixes #65434
elastic · Nov 26, 2020 · fb84b67 · fb84b67
1 parent f6921af
commit fb84b67
Show file tree

Hide file tree

Showing 20 changed files with 286 additions and 94 deletions.
diff --git a/...tTest/resources/rest-api-spec/test/search.query/41_query_string_with_default_analyzer.yml b/...tTest/resources/rest-api-spec/test/search.query/41_query_string_with_default_analyzer.yml
@@ -0,0 +1,39 @@
+---
+"Test default search analyzer is applied":
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            index.analysis.analyzer.default.type: simple
+            index.analysis.analyzer.default_search.type: german
+          mappings:
+            properties:
+              body:
+                type: text
+
+  - do:
+      index:
+        index: test
+        id: 1
+        body:
+          body: Ich lese die Bücher
+
+  - do:
+      indices.refresh:
+        index: [ test ]
+
+  - do:
+      search:
+        index: test
+        q: "body:Bücher"
+
+  - match: { hits.total.value: 0 }
+
+  - do:
+      search:
+        index: test
+        q: "body:Bücher"
+        analyzer: simple
+
+  - match: { hits.total.value: 1 }
diff --git a/...apper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java b/...apper-extras/src/main/java/org/elasticsearch/index/mapper/SearchAsYouTypeFieldMapper.java
@@ -48,6 +48,7 @@
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.common.collect.Iterators;
 import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.query.QueryShardContext;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -61,7 +62,6 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.function.Supplier;
 
 import static org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType.hasGaps;
 
@@ -93,7 +93,7 @@ public static class Defaults {
     }
 
     public static final TypeParser PARSER
-        = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
+        = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
 
     private static SearchAsYouTypeFieldMapper toType(FieldMapper in) {
         return (SearchAsYouTypeFieldMapper) in;
@@ -138,9 +138,9 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
 
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
-        public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
+        public Builder(String name, IndexAnalyzers indexAnalyzers) {
             super(name);
-            this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
+            this.analyzers = new TextParams.Analyzers(indexAnalyzers);
         }
 
         @Override
@@ -592,7 +592,7 @@ protected String contentType() {
 
     @Override
     public ParametrizedFieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
+        return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
     }
 
     public static String getShingleFieldName(String parentField, int shingleSize) {

diff --git a/.../src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java b/.../src/main/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldMapper.java
@@ -33,6 +33,7 @@
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.FieldMapper;
 import org.elasticsearch.index.mapper.MapperParsingException;
@@ -55,7 +56,6 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Supplier;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -102,9 +102,9 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
         private final Parameter<Float> boost = Parameter.boostParam();
         private final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
-        public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
+        public Builder(String name, IndexAnalyzers indexAnalyzers) {
             super(name);
-            this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
+            this.analyzers = new TextParams.Analyzers(indexAnalyzers);
         }
 
         @Override
@@ -157,7 +157,7 @@ name, fieldType, buildFieldType(fieldType, context),
         }
     }
 
-    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
+    public static TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getIndexAnalyzers()));
 
     /**
      * Parses markdown-like syntax into plain text and AnnotationTokens with offsets for
@@ -571,6 +571,6 @@ protected String contentType() {
 
     @Override
     public ParametrizedFieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
+        return new Builder(simpleName(), builder.analyzers.indexAnalyzers).init(this);
     }
 }
diff --git a/...c/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java b/...c/test/java/org/elasticsearch/index/mapper/annotatedtext/AnnotatedTextFieldTypeTests.java
@@ -22,9 +22,6 @@
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.queries.intervals.Intervals;
 import org.apache.lucene.queries.intervals.IntervalsSource;
-import org.elasticsearch.Version;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.analysis.AnalyzerScope;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
@@ -46,11 +43,8 @@ public void testIntervals() throws IOException {
     }
 
     public void testFetchSourceValue() throws IOException {
-        Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT.id).build();
-        Mapper.BuilderContext context = new Mapper.BuilderContext(settings, new ContentPath());
-
-        MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", () -> Lucene.STANDARD_ANALYZER)
-            .build(context)
+        MappedFieldType fieldType = new AnnotatedTextFieldMapper.Builder("field", createDefaultIndexAnalyzers())
+            .build(new Mapper.BuilderContext(Settings.EMPTY, new ContentPath()))
             .fieldType();
 
         assertEquals(Collections.singletonList("value"), fetchSourceValue(fieldType, "value"));

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java
@@ -703,8 +703,7 @@ private static Mapper.Builder<?> createBuilderFromDynamicValue(final ParseContex
 
             Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING);
             if (builder == null) {
-                builder = new TextFieldMapper.Builder(currentFieldName,
-                    () -> context.mapperService().getIndexAnalyzers().getDefaultIndexAnalyzer())
+                builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers())
                         .addMultiField(new KeywordFieldMapper.Builder("keyword").ignoreAbove(256));
             }
             return builder;

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java
@@ -65,6 +65,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.index.analysis.AnalyzerScope;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.index.fielddata.plain.PagedBytesIndexFieldData;
@@ -271,14 +272,14 @@ public static class Builder extends ParametrizedFieldMapper.Builder {
 
         final TextParams.Analyzers analyzers;
 
-        public Builder(String name, Supplier<NamedAnalyzer> defaultAnalyzer) {
-            this(name, Version.CURRENT, defaultAnalyzer);
+        public Builder(String name, IndexAnalyzers indexAnalyzers) {
+            this(name, Version.CURRENT, indexAnalyzers);
         }
 
-        public Builder(String name, Version indexCreatedVersion, Supplier<NamedAnalyzer> defaultAnalyzer) {
+        public Builder(String name, Version indexCreatedVersion, IndexAnalyzers indexAnalyzers) {
             super(name);
             this.indexCreatedVersion = indexCreatedVersion;
-            this.analyzers = new TextParams.Analyzers(defaultAnalyzer);
+            this.analyzers = new TextParams.Analyzers(indexAnalyzers);
         }
 
         public Builder index(boolean index) {
@@ -400,7 +401,7 @@ public TextFieldMapper build(BuilderContext context) {
     }
 
     public static final TypeParser PARSER
-        = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), () -> c.getIndexAnalyzers().getDefaultIndexAnalyzer()));
+        = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers()));
 
     private static class PhraseWrappedAnalyzer extends AnalyzerWrapper {
 
@@ -862,7 +863,7 @@ protected TextFieldMapper clone() {
 
     @Override
     public ParametrizedFieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzer::getDefaultValue).init(this);
+        return new Builder(simpleName(), builder.indexCreatedVersion, builder.analyzers.indexAnalyzers).init(this);
     }
 
     @Override

diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java b/server/src/main/java/org/elasticsearch/index/mapper/TextParams.java
@@ -22,6 +22,8 @@
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.index.IndexOptions;
 import org.elasticsearch.index.analysis.AnalysisMode;
+import org.elasticsearch.index.analysis.AnalysisRegistry;
+import org.elasticsearch.index.analysis.IndexAnalyzers;
 import org.elasticsearch.index.analysis.NamedAnalyzer;
 import org.elasticsearch.index.mapper.ParametrizedFieldMapper.Parameter;
 import org.elasticsearch.index.similarity.SimilarityProvider;
@@ -42,21 +44,37 @@ public static final class Analyzers {
         public final Parameter<NamedAnalyzer> searchAnalyzer;
         public final Parameter<NamedAnalyzer> searchQuoteAnalyzer;
 
-        public Analyzers(Supplier<NamedAnalyzer> defaultAnalyzer) {
+        public final IndexAnalyzers indexAnalyzers;
+
+        public Analyzers(IndexAnalyzers indexAnalyzers) {
             this.indexAnalyzer = Parameter.analyzerParam("analyzer", false,
-                m -> m.fieldType().indexAnalyzer(), defaultAnalyzer)
+                m -> m.fieldType().indexAnalyzer(), indexAnalyzers::getDefaultIndexAnalyzer)
                 .setSerializerCheck((id, ic, a) -> id || ic ||
                     Objects.equals(a, getSearchAnalyzer()) == false || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
                 .setValidator(a -> a.checkAllowedInMode(AnalysisMode.INDEX_TIME));
             this.searchAnalyzer
                 = Parameter.analyzerParam("search_analyzer", true,
-                m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), indexAnalyzer::getValue)
+                m -> m.fieldType().getTextSearchInfo().getSearchAnalyzer(), () -> {
+                    NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_ANALYZER_NAME);
+                    if (defaultAnalyzer != null) {
+                        return defaultAnalyzer;
+                    }
+                    return indexAnalyzer.get();
+                })
                 .setSerializerCheck((id, ic, a) -> id || ic || Objects.equals(a, getSearchQuoteAnalyzer()) == false)
                 .setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
             this.searchQuoteAnalyzer
                 = Parameter.analyzerParam("search_quote_analyzer", true,
-                m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), searchAnalyzer::getValue)
+                m -> m.fieldType().getTextSearchInfo().getSearchQuoteAnalyzer(), () -> {
+                    NamedAnalyzer defaultAnalyzer = indexAnalyzers.get(AnalysisRegistry.DEFAULT_SEARCH_QUOTED_ANALYZER_NAME);
+                    if (defaultAnalyzer != null) {
+                        return defaultAnalyzer;
+                    }
+                    return searchAnalyzer.get();
+                })
                 .setValidator(a -> a.checkAllowedInMode(AnalysisMode.SEARCH_TIME));
+
+            this.indexAnalyzers = indexAnalyzers;
         }
 
         public NamedAnalyzer getIndexAnalyzer() {

diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java
@@ -297,7 +297,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap
             return fieldMapping;
         } else if (mapUnmappedFieldAsString) {
             TextFieldMapper.Builder builder
-                = new TextFieldMapper.Builder(name, () -> mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer());
+                = new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
             return builder.build(new Mapper.BuilderContext(indexSettings.getSettings(), new ContentPath(1))).fieldType();
         } else {
             throw new QueryShardException(this, "No field mapping can be found for the field with name [{}]", name);

diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java b/server/src/test/java/org/elasticsearch/index/fielddata/AbstractFieldDataTestCase.java
@@ -31,7 +31,6 @@
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.ByteBuffersDirectory;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
@@ -100,7 +99,7 @@ public <IFD extends IndexFieldData<?>> IFD getForField(String type, String field
             if (docValues) {
                 fieldType = new KeywordFieldMapper.Builder(fieldName).build(context).fieldType();
             } else {
-                fieldType = new TextFieldMapper.Builder(fieldName, () -> Lucene.STANDARD_ANALYZER)
+                fieldType = new TextFieldMapper.Builder(fieldName, createDefaultIndexAnalyzers())
                     .fielddata(true).build(context).fieldType();
             }
         } else if (type.equals("float")) {

diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/FilterFieldDataTests.java
@@ -23,7 +23,6 @@
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.SortedSetDocValues;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.Mapper.BuilderContext;
@@ -65,7 +64,7 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
+            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
                     .fielddata(true)
                     .fielddataFrequencyFilter(0, random.nextBoolean() ? 100 : 0.5d, 0)
                     .build(builderCtx).fieldType();
@@ -80,7 +79,7 @@ public void testFilterByFrequency() throws Exception {
         }
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", () -> Lucene.STANDARD_ANALYZER)
+            MappedFieldType ft = new TextFieldMapper.Builder("high_freq", createDefaultIndexAnalyzers())
                     .fielddata(true)
                     .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, 201, 100)
                     .build(builderCtx).fieldType();
@@ -95,7 +94,7 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);// test # docs with value
-            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
+            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
                     .fielddata(true)
                     .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
                     .build(builderCtx).fieldType();
@@ -111,7 +110,7 @@ public void testFilterByFrequency() throws Exception {
 
         {
             indexService.clearCaches(false, true);
-            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", () -> Lucene.STANDARD_ANALYZER)
+            MappedFieldType ft = new TextFieldMapper.Builder("med_freq", createDefaultIndexAnalyzers())
                     .fielddata(true)
                     .fielddataFrequencyFilter(random.nextBoolean() ? 101 : 101d/200.0d, Integer.MAX_VALUE, 101)
                     .build(builderCtx).fieldType();
@@ -128,7 +127,7 @@ public void testFilterByFrequency() throws Exception {
     }
 
     @Override
-    public void testEmpty() throws Exception {
+    public void testEmpty() {
         assumeTrue("No need to test empty usage here", false);
     }
 }
diff --git a/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java b/server/src/test/java/org/elasticsearch/index/fielddata/IndexFieldDataServiceTests.java
@@ -31,7 +31,6 @@
 import org.apache.lucene.store.ByteBuffersDirectory;
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.SetOnce;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexService;
@@ -143,9 +142,9 @@ public void testClearField() throws Exception {
 
         final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
         final MappedFieldType mapper1
-            = new TextFieldMapper.Builder("field_1", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
+            = new TextFieldMapper.Builder("field_1", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
         final MappedFieldType mapper2
-            = new TextFieldMapper.Builder("field_2", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
+            = new TextFieldMapper.Builder("field_2", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
         doc.add(new StringField("field_1", "thisisastring", Store.NO));
@@ -209,7 +208,7 @@ public void testFieldDataCacheListener() throws Exception {
 
         final BuilderContext ctx = new BuilderContext(indexService.getIndexSettings().getSettings(), new ContentPath(1));
         final MappedFieldType mapper1
-            = new TextFieldMapper.Builder("s", () -> Lucene.STANDARD_ANALYZER).fielddata(true).build(ctx).fieldType();
+            = new TextFieldMapper.Builder("s", createDefaultIndexAnalyzers()).fielddata(true).build(ctx).fieldType();
         final IndexWriter writer = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new KeywordAnalyzer()));
         Document doc = new Document();
         doc.add(new StringField("s", "thisisastring", Store.NO));