Search old indices using doc values (#84193)

Indices older than N-1 do not currently provide access to their index structures. Searches can only run using 
doc-values. Instead of requiring users (or a mapping auto-conversion) to explicitly set index:false on such fields, we
automatically run queries using doc-values on such indices.

modules/percolator/src/main/java/org/elasticsearch/percolator/PercolatorFieldMapper.java

@@ -76,6 +76,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.function.Supplier;
 
 import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder;
@@ -102,7 +103,7 @@ public class PercolatorFieldMapper extends FieldMapper {
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), searchExecutionContext, mapUnmappedFieldsAsText).init(this);
+        return new Builder(simpleName(), searchExecutionContext, mapUnmappedFieldsAsText, indexCreatedVersion).init(this);
     }
 
     static class Builder extends FieldMapper.Builder {
@@ -112,10 +113,18 @@ static class Builder extends FieldMapper.Builder {
         private final Supplier<SearchExecutionContext> searchExecutionContext;
         private final boolean mapUnmappedFieldsAsText;
 
-        Builder(String fieldName, Supplier<SearchExecutionContext> searchExecutionContext, boolean mapUnmappedFieldsAsText) {
+        private final Version indexCreatedVersion;
+
+        Builder(
+            String fieldName,
+            Supplier<SearchExecutionContext> searchExecutionContext,
+            boolean mapUnmappedFieldsAsText,
+            Version indexCreatedVersion
+        ) {
             super(fieldName);
             this.searchExecutionContext = searchExecutionContext;
             this.mapUnmappedFieldsAsText = mapUnmappedFieldsAsText;
+            this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
         }
 
         @Override
@@ -129,17 +138,25 @@ public PercolatorFieldMapper build(MapperBuilderContext context) {
             // TODO should percolator even allow multifields?
             MultiFields multiFields = multiFieldsBuilder.build(this, context);
             context = context.createChildContext(name);
-            KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(EXTRACTED_TERMS_FIELD_NAME, context);
+            KeywordFieldMapper extractedTermsField = createExtractQueryFieldBuilder(
+                EXTRACTED_TERMS_FIELD_NAME,
+                context,
+                indexCreatedVersion
+            );
             fieldType.queryTermsField = extractedTermsField.fieldType();
-            KeywordFieldMapper extractionResultField = createExtractQueryFieldBuilder(EXTRACTION_RESULT_FIELD_NAME, context);
+            KeywordFieldMapper extractionResultField = createExtractQueryFieldBuilder(
+                EXTRACTION_RESULT_FIELD_NAME,
+                context,
+                indexCreatedVersion
+            );
             fieldType.extractionResultField = extractionResultField.fieldType();
             BinaryFieldMapper queryBuilderField = createQueryBuilderFieldBuilder(context);
             fieldType.queryBuilderField = queryBuilderField.fieldType();
             // Range field is of type ip, because that matches closest with BinaryRange field. Otherwise we would
             // have to introduce a new field type...
             RangeFieldMapper rangeFieldMapper = createExtractedRangeFieldBuilder(RANGE_FIELD_NAME, RangeType.IP, context);
             fieldType.rangeField = rangeFieldMapper.fieldType();
-            NumberFieldMapper minimumShouldMatchFieldMapper = createMinimumShouldMatchField(context);
+            NumberFieldMapper minimumShouldMatchFieldMapper = createMinimumShouldMatchField(context, indexCreatedVersion);
             fieldType.minimumShouldMatchField = minimumShouldMatchFieldMapper.fieldType();
             fieldType.mapUnmappedFieldsAsText = mapUnmappedFieldsAsText;
 
@@ -154,12 +171,13 @@ public PercolatorFieldMapper build(MapperBuilderContext context) {
                 queryBuilderField,
                 rangeFieldMapper,
                 minimumShouldMatchFieldMapper,
-                mapUnmappedFieldsAsText
+                mapUnmappedFieldsAsText,
+                indexCreatedVersion
             );
         }
 
-        static KeywordFieldMapper createExtractQueryFieldBuilder(String name, MapperBuilderContext context) {
-            KeywordFieldMapper.Builder queryMetadataFieldBuilder = new KeywordFieldMapper.Builder(name);
+        static KeywordFieldMapper createExtractQueryFieldBuilder(String name, MapperBuilderContext context, Version indexCreatedVersion) {
+            KeywordFieldMapper.Builder queryMetadataFieldBuilder = new KeywordFieldMapper.Builder(name, indexCreatedVersion);
             queryMetadataFieldBuilder.docValues(false);
             return queryMetadataFieldBuilder.build(context);
         }
@@ -176,10 +194,11 @@ static RangeFieldMapper createExtractedRangeFieldBuilder(String name, RangeType
             return builder.build(context);
         }
 
-        static NumberFieldMapper createMinimumShouldMatchField(MapperBuilderContext context) {
+        static NumberFieldMapper createMinimumShouldMatchField(MapperBuilderContext context, Version indexCreatedVersion) {
             NumberFieldMapper.Builder builder = NumberFieldMapper.Builder.docValuesOnly(
                 MINIMUM_SHOULD_MATCH_FIELD_NAME,
-                NumberFieldMapper.NumberType.INTEGER
+                NumberFieldMapper.NumberType.INTEGER,
+                indexCreatedVersion
             );
             return builder.build(context);
         }
@@ -190,7 +209,12 @@ static class TypeParser implements Mapper.TypeParser {
 
         @Override
         public Builder parse(String name, Map<String, Object> node, MappingParserContext parserContext) throws MapperParsingException {
-            return new Builder(name, parserContext.searchExecutionContext(), getMapUnmappedFieldAsText(parserContext.getSettings()));
+            return new Builder(
+                name,
+                parserContext.searchExecutionContext(),
+                getMapUnmappedFieldAsText(parserContext.getSettings()),
+                parserContext.indexVersionCreated()
+            );
         }
     }
 
@@ -336,6 +360,7 @@ Tuple<List<BytesRef>, Map<String, List<byte[]>>> extractTermsAndRanges(IndexRead
     private final NumberFieldMapper minimumShouldMatchFieldMapper;
     private final RangeFieldMapper rangeFieldMapper;
     private final boolean mapUnmappedFieldsAsText;
+    private final Version indexCreatedVersion;
 
     PercolatorFieldMapper(
         String simpleName,
@@ -348,7 +373,8 @@ Tuple<List<BytesRef>, Map<String, List<byte[]>>> extractTermsAndRanges(IndexRead
         BinaryFieldMapper queryBuilderField,
         RangeFieldMapper rangeFieldMapper,
         NumberFieldMapper minimumShouldMatchFieldMapper,
-        boolean mapUnmappedFieldsAsText
+        boolean mapUnmappedFieldsAsText,
+        Version indexCreatedVersion
     ) {
         super(simpleName, mappedFieldType, multiFields, copyTo);
         this.searchExecutionContext = searchExecutionContext;
@@ -358,6 +384,7 @@ Tuple<List<BytesRef>, Map<String, List<byte[]>>> extractTermsAndRanges(IndexRead
         this.minimumShouldMatchFieldMapper = minimumShouldMatchFieldMapper;
         this.rangeFieldMapper = rangeFieldMapper;
         this.mapUnmappedFieldsAsText = mapUnmappedFieldsAsText;
+        this.indexCreatedVersion = indexCreatedVersion;
     }
 
     @Override

server/src/main/java/org/elasticsearch/Version.java

@@ -386,6 +386,14 @@ private Version computeMinIndexCompatVersion() {
         return Version.min(this, fromId(bwcMajor * 1000000 + bwcMinor * 10000 + 99));
     }
 
+    /**
+     * Whether the current version is older than the current minimum compatible index version,
+     * see {@link #minimumIndexCompatibilityVersion()}
+     */
+    public boolean isLegacyIndexVersion() {
+        return before(Version.CURRENT.minimumIndexCompatibilityVersion());
+    }
+
     /**
      * Returns <code>true</code> iff both version are compatible. Otherwise <code>false</code>
      */

server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java

@@ -1999,8 +1999,7 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
                 } else if (token == XContentParser.Token.START_OBJECT) {
                     if ("settings".equals(currentFieldName)) {
                         Settings settings = Settings.fromXContent(parser);
-                        if (SETTING_INDEX_VERSION_COMPATIBILITY.get(settings)
-                            .onOrAfter(Version.CURRENT.minimumIndexCompatibilityVersion())) {
+                        if (SETTING_INDEX_VERSION_COMPATIBILITY.get(settings).isLegacyIndexVersion() == false) {
                             throw new IllegalStateException(
                                 "this method should only be used to parse older incompatible index metadata versions "
                                     + "but got "
@@ -2085,7 +2084,8 @@ public static IndexMetadata legacyFromXContent(XContentParser parser) throws IOE
             }
 
             IndexMetadata indexMetadata = builder.build();
-            assert indexMetadata.getCompatibilityVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion());
+            assert indexMetadata.getCreationVersion().isLegacyIndexVersion();
+            assert indexMetadata.getCompatibilityVersion().isLegacyIndexVersion();
             return indexMetadata;
         }
 

server/src/main/java/org/elasticsearch/env/NodeEnvironment.java

@@ -501,7 +501,7 @@ static void checkForIndexCompatibility(Logger logger, NodePath... nodePaths) thr
 
         logger.info("oldest index version recorded in NodeMetadata {}", metadata.oldestIndexVersion());
 
-        if (metadata.oldestIndexVersion().before(Version.CURRENT.minimumIndexCompatibilityVersion())) {
+        if (metadata.oldestIndexVersion().isLegacyIndexVersion()) {
             throw new IllegalStateException(
                 "cannot upgrade node because incompatible indices created with version ["
                     + metadata.oldestIndexVersion()

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -21,6 +21,7 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermRangeQuery;
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
 import org.elasticsearch.core.Booleans;
@@ -98,9 +99,12 @@ public static class Builder extends FieldMapper.Builder {
 
         private final ScriptCompiler scriptCompiler;
 
-        public Builder(String name, ScriptCompiler scriptCompiler) {
+        private final Version indexCreatedVersion;
+
+        public Builder(String name, ScriptCompiler scriptCompiler, Version indexCreatedVersion) {
             super(name);
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
+            this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
             this.script.precludesParameters(nullValue);
             addScriptValidation(script, indexed, docValues);
         }
@@ -114,7 +118,7 @@ protected List<Parameter<?>> getParameters() {
         public BooleanFieldMapper build(MapperBuilderContext context) {
             MappedFieldType ft = new BooleanFieldType(
                 context.buildFullName(name),
-                indexed.getValue(),
+                indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false,
                 stored.getValue(),
                 docValues.getValue(),
                 nullValue.getValue(),
@@ -138,7 +142,7 @@ private FieldValues<Boolean> scriptValues() {
         }
     }
 
-    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler()));
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.scriptCompiler(), c.indexVersionCreated()));
 
     public static final class BooleanFieldType extends TermBasedFieldType {
 
@@ -328,6 +332,7 @@ public Query rangeQuery(
     private final Script script;
     private final FieldValues<Boolean> scriptValues;
     private final ScriptCompiler scriptCompiler;
+    private final Version indexCreatedVersion;
 
     protected BooleanFieldMapper(
         String simpleName,
@@ -352,6 +357,7 @@ protected BooleanFieldMapper(
         this.script = builder.script.get();
         this.scriptValues = builder.scriptValues();
         this.scriptCompiler = builder.scriptCompiler;
+        this.indexCreatedVersion = builder.indexCreatedVersion;
     }
 
     @Override
@@ -406,7 +412,7 @@ protected void indexScriptValues(
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), scriptCompiler).init(this);
+        return new Builder(simpleName(), scriptCompiler, indexCreatedVersion).init(this);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -326,7 +326,7 @@ private Long parseNullValue(DateFieldType fieldType) {
         public DateFieldMapper build(MapperBuilderContext context) {
             DateFieldType ft = new DateFieldType(
                 context.buildFullName(name()),
-                index.getValue(),
+                index.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false,
                 store.getValue(),
                 docValues.getValue(),
                 buildFormatter(),

server/src/main/java/org/elasticsearch/index/mapper/DynamicFieldsBuilder.java

@@ -311,7 +311,7 @@ void createDynamicField(Mapper.Builder builder, DocumentParserContext context) t
         public void newDynamicStringField(DocumentParserContext context, String name) throws IOException {
             createDynamicField(
                 new TextFieldMapper.Builder(name, context.indexAnalyzers()).addMultiField(
-                    new KeywordFieldMapper.Builder("keyword").ignoreAbove(256)
+                    new KeywordFieldMapper.Builder("keyword", context.indexSettings().getIndexVersionCreated()).ignoreAbove(256)
                 ),
                 context
             );
@@ -324,7 +324,8 @@ public void newDynamicLongField(DocumentParserContext context, String name) thro
                     name,
                     NumberFieldMapper.NumberType.LONG,
                     ScriptCompiler.NONE,
-                    context.indexSettings().getSettings()
+                    context.indexSettings().getSettings(),
+                    context.indexSettings().getIndexVersionCreated()
                 ),
                 context
             );
@@ -340,15 +341,19 @@ public void newDynamicDoubleField(DocumentParserContext context, String name) th
                     name,
                     NumberFieldMapper.NumberType.FLOAT,
                     ScriptCompiler.NONE,
-                    context.indexSettings().getSettings()
+                    context.indexSettings().getSettings(),
+                    context.indexSettings().getIndexVersionCreated()
                 ),
                 context
             );
         }
 
         @Override
         public void newDynamicBooleanField(DocumentParserContext context, String name) throws IOException {
-            createDynamicField(new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE), context);
+            createDynamicField(
+                new BooleanFieldMapper.Builder(name, ScriptCompiler.NONE, context.indexSettings().getIndexVersionCreated()),
+                context
+            );
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -17,6 +17,7 @@
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.Version;
 import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.Explicit;
 import org.elasticsearch.common.geo.GeoFormatterFactory;
@@ -82,8 +83,9 @@ public static class Builder extends FieldMapper.Builder {
         final Parameter<Map<String, String>> meta = Parameter.metaParam();
 
         private final ScriptCompiler scriptCompiler;
+        private final Version indexCreatedVersion;
 
-        public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault) {
+        public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalformedByDefault, Version indexCreatedVersion) {
             super(name);
             this.ignoreMalformed = ignoreMalformedParam(m -> builder(m).ignoreMalformed.get(), ignoreMalformedByDefault);
             this.nullValue = nullValueParam(
@@ -93,6 +95,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, boolean ignoreMalform
                 XContentBuilder::field
             ).acceptsNull();
             this.scriptCompiler = Objects.requireNonNull(scriptCompiler);
+            this.indexCreatedVersion = Objects.requireNonNull(indexCreatedVersion);
             this.script.precludesParameters(nullValue, ignoreMalformed, ignoreZValue);
             addScriptValidation(script, indexed, hasDocValues);
         }
@@ -146,7 +149,7 @@ public FieldMapper build(MapperBuilderContext context) {
             }, nullValue.get(), ignoreZValue.get().value(), ignoreMalformed.get().value());
             GeoPointFieldType ft = new GeoPointFieldType(
                 context.buildFullName(name),
-                indexed.get(),
+                indexed.get() && indexCreatedVersion.isLegacyIndexVersion() == false,
                 stored.get(),
                 hasDocValues.get(),
                 geoParser,
@@ -162,11 +165,12 @@ public FieldMapper build(MapperBuilderContext context) {
     }
 
     public static TypeParser PARSER = new TypeParser(
-        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()))
+        (n, c) -> new Builder(n, c.scriptCompiler(), IGNORE_MALFORMED_SETTING.get(c.getSettings()), c.indexVersionCreated())
     );
 
     private final Builder builder;
     private final FieldValues<GeoPoint> scriptValues;
+    private final Version indexCreatedVersion;
 
     public GeoPointFieldMapper(
         String simpleName,
@@ -188,17 +192,20 @@ public GeoPointFieldMapper(
         );
         this.builder = builder;
         this.scriptValues = null;
+        this.indexCreatedVersion = builder.indexCreatedVersion;
     }
 
     public GeoPointFieldMapper(String simpleName, MappedFieldType mappedFieldType, Parser<GeoPoint> parser, Builder builder) {
         super(simpleName, mappedFieldType, MultiFields.empty(), CopyTo.empty(), parser, builder.onScriptError.get());
         this.builder = builder;
         this.scriptValues = builder.scriptValues();
+        this.indexCreatedVersion = builder.indexCreatedVersion;
     }
 
     @Override
     public FieldMapper.Builder getMergeBuilder() {
-        return new Builder(simpleName(), builder.scriptCompiler, builder.ignoreMalformed.getDefaultValue().value()).init(this);
+        return new Builder(simpleName(), builder.scriptCompiler, builder.ignoreMalformed.getDefaultValue().value(), indexCreatedVersion)
+            .init(this);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -159,7 +159,7 @@ public IpFieldMapper build(MapperBuilderContext context) {
                 name,
                 new IpFieldType(
                     context.buildFullName(name),
-                    indexed.getValue(),
+                    indexed.getValue() && indexCreatedVersion.isLegacyIndexVersion() == false,
                     stored.getValue(),
                     hasDocValues.getValue(),
                     parseNullValue(),