Use docvalue skippers on dimension fields

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.breaker.NoopCircuitBreaker;
 import org.elasticsearch.common.logging.LogConfigurator;
-import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.util.BigArrays;
@@ -227,50 +226,44 @@ private static BlockLoader blockLoader(String name) {
                     break;
             }
             ft.freeze();
-            return new KeywordFieldMapper.KeywordFieldType(
-                w.name,
-                ft,
-                Lucene.KEYWORD_ANALYZER,
-                Lucene.KEYWORD_ANALYZER,
-                Lucene.KEYWORD_ANALYZER,
-                new KeywordFieldMapper.Builder(name, defaultIndexSettings()).docValues(ft.docValuesType() != DocValuesType.NONE),
-                syntheticSource
-            ).blockLoader(new MappedFieldType.BlockLoaderContext() {
-                @Override
-                public String indexName() {
-                    return "benchmark";
-                }
+            return new KeywordFieldMapper.KeywordFieldType(w.name, ft, syntheticSource).blockLoader(
+                new MappedFieldType.BlockLoaderContext() {
+                    @Override
+                    public String indexName() {
+                        return "benchmark";
+                    }
 
-                @Override
-                public IndexSettings indexSettings() {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public IndexSettings indexSettings() {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
-                    return MappedFieldType.FieldExtractPreference.NONE;
-                }
+                    @Override
+                    public MappedFieldType.FieldExtractPreference fieldExtractPreference() {
+                        return MappedFieldType.FieldExtractPreference.NONE;
+                    }
 
-                @Override
-                public SearchLookup lookup() {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public SearchLookup lookup() {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public Set<String> sourcePaths(String name) {
-                    return Set.of(name);
-                }
+                    @Override
+                    public Set<String> sourcePaths(String name) {
+                        return Set.of(name);
+                    }
 
-                @Override
-                public String parentField(String field) {
-                    throw new UnsupportedOperationException();
-                }
+                    @Override
+                    public String parentField(String field) {
+                        throw new UnsupportedOperationException();
+                    }
 
-                @Override
-                public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
-                    return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
+                    @Override
+                    public FieldNamesFieldMapper.FieldNamesFieldType fieldNames() {
+                        return FieldNamesFieldMapper.FieldNamesFieldType.get(true);
+                    }
                 }
-            });
+            );
         }
         throw new IllegalArgumentException("can't read [" + name + "]");
     }

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/ScaledFloatFieldMapper.java

@@ -722,7 +722,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
         }
         long scaledValue = encode(doubleValue, scalingFactor);
 
-        NumberFieldMapper.NumberType.LONG.addFields(context.doc(), fieldType().name(), scaledValue, indexed, hasDocValues, stored);
+        NumberFieldMapper.NumberType.LONG.addFields(
+            context.doc(),
+            fieldType().name(),
+            scaledValue,
+            IndexType.points(indexed, hasDocValues),
+            stored
+        );
 
         if (shouldStoreOffsets) {
             context.getOffSetContext().recordOffset(offsetsFieldName, scaledValue);

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java

@@ -164,7 +164,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
             tokenCount = countPositions(analyzer, fullPath(), value, enablePositionIncrements);
         }
 
-        NumberFieldMapper.NumberType.INTEGER.addFields(context.doc(), fieldType().name(), tokenCount, index, hasDocValues, store);
+        NumberFieldMapper.NumberType.INTEGER.addFields(
+            context.doc(),
+            fieldType().name(),
+            tokenCount,
+            IndexType.points(index, hasDocValues),
+            store
+        );
     }
 
     /**

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldTypeTests.java

@@ -42,6 +42,7 @@
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
 import org.elasticsearch.index.mapper.FieldTypeTestCase;
+import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MappingParserContext;
@@ -295,9 +296,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet
 
         KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType(
             "child",
-            mock(FieldType.class),
-            mock(NamedAnalyzer.class),
-            mock(NamedAnalyzer.class),
+            IndexType.terms(true, true),
+            new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)),
             mock(NamedAnalyzer.class),
             builder,
             true
@@ -344,9 +344,8 @@ public void testBlockLoaderDoesNotUseSyntheticSourceDelegateWhenIgnoreAboveIsSet
 
         KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType(
             "child",
-            mock(FieldType.class),
-            mock(NamedAnalyzer.class),
-            mock(NamedAnalyzer.class),
+            IndexType.terms(true, true),
+            new TextSearchInfo(mock(FieldType.class), null, mock(NamedAnalyzer.class), mock(NamedAnalyzer.class)),
             mock(NamedAnalyzer.class),
             builder,
             true

modules/percolator/src/test/java/org/elasticsearch/percolator/CandidateQueryTests.java

@@ -79,6 +79,7 @@
 import org.elasticsearch.index.IndexService;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.DocumentParserContext;
+import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.LuceneDocument;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.MapperService;
@@ -301,7 +302,7 @@ public void testDuel() throws Exception {
                 document.add(new TextField(entry.getKey(), value, Field.Store.NO));
             }
             for (Integer intValue : intValues) {
-                NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, true, true, false);
+                NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", intValue, IndexType.points(true, true), false);
             }
             MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());
             duelRun(queryStore, memoryIndex, shardSearcher);
@@ -424,7 +425,13 @@ public void testDuel2() throws Exception {
             }
 
             for (int[] range : ranges) {
-                NumberFieldMapper.NumberType.INTEGER.addFields(document, "int_field", between(range[0], range[1]), true, true, false);
+                NumberFieldMapper.NumberType.INTEGER.addFields(
+                    document,
+                    "int_field",
+                    between(range[0], range[1]),
+                    IndexType.points(true, true),
+                    false
+                );
                 logger.info("Test with document: {}" + document);
                 MemoryIndex memoryIndex = MemoryIndex.fromDocument(document, new WhitespaceAnalyzer());
                 duelRun(queryStore, memoryIndex, shardSearcher);

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -104,7 +104,7 @@ public void postParse(DocumentParserContext context) {
             return;
         }
         final int value = context.sourceToParse().source().length();
-        NumberType.INTEGER.addFields(context.doc(), fullPath(), value, true, true, true);
+        NumberType.INTEGER.addFields(context.doc(), fullPath(), value, IndexType.points(true, true), true);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -192,6 +192,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion REENABLED_TIMESTAMP_DOC_VALUES_SPARSE_INDEX = def(9_042_0_00, Version.LUCENE_10_3_1);
     public static final IndexVersion SKIPPERS_ENABLED_BY_DEFAULT = def(9_043_0_00, Version.LUCENE_10_3_1);
     public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID = def(9_044_0_00, Version.LUCENE_10_3_1);
+    public static final IndexVersion TIME_SERIES_DIMENSIONS_USE_SKIPPERS = def(9_045_0_00, Version.LUCENE_10_3_1);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -79,7 +79,7 @@ private static BooleanFieldMapper toType(FieldMapper in) {
     public static final class Builder extends FieldMapper.DimensionBuilder {
 
         private final Parameter<Boolean> docValues = Parameter.docValuesParam(m -> toType(m).hasDocValues, true);
-        private final Parameter<Boolean> indexed = Parameter.indexParam(m -> toType(m).indexed, true);
+        private final Parameter<Boolean> indexed;
         private final Parameter<Boolean> stored = Parameter.storeParam(m -> toType(m).stored, false);
         private final Parameter<Explicit<Boolean>> ignoreMalformed;
         private final Parameter<Boolean> nullValue = new Parameter<>(
@@ -117,20 +117,9 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe
                 IGNORE_MALFORMED_SETTING.get(indexSettings.getSettings())
             );
             this.script.precludesParameters(ignoreMalformed, nullValue);
+            this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension(), docValues::get);
+            this.indexed = Parameter.indexParam(m -> toType(m).indexed, indexSettings, dimension);
             addScriptValidation(script, indexed, docValues);
-            this.dimension = TimeSeriesParams.dimensionParam(m -> toType(m).fieldType().isDimension()).addValidator(v -> {
-                if (v && (indexed.getValue() == false || docValues.getValue() == false)) {
-                    throw new IllegalArgumentException(
-                        "Field ["
-                            + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM
-                            + "] requires that ["
-                            + indexed.name
-                            + "] and ["
-                            + docValues.name
-                            + "] are true"
-                    );
-                }
-            });
         }
 
         public Builder dimension(boolean dimension) {
@@ -152,16 +141,30 @@ protected Parameter<?>[] getParameters() {
                 dimension };
         }
 
+        private IndexType indexType() {
+            if (indexed.get() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false) {
+                return IndexType.terms(true, docValues.getValue());
+            }
+            if (docValues.get() == false) {
+                return IndexType.NONE;
+            }
+            if (dimension.get()
+                && indexSettings.useDocValuesSkipper()
+                && indexSettings.getIndexVersionCreated().onOrAfter(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)) {
+                return IndexType.skippers();
+            }
+            return IndexType.docValuesOnly();
+        }
+
         @Override
         public BooleanFieldMapper build(MapperBuilderContext context) {
             if (inheritDimensionParameterFromParentObject(context)) {
                 dimension(true);
             }
             MappedFieldType ft = new BooleanFieldType(
                 context.buildFullName(leafName()),
-                indexed.getValue() && indexSettings.getIndexVersionCreated().isLegacyIndexVersion() == false,
+                indexType(),
                 stored.getValue(),
-                docValues.getValue(),
                 nullValue.getValue(),
                 scriptValues(),
                 meta.getValue(),
@@ -215,32 +218,27 @@ public static final class BooleanFieldType extends TermBasedFieldType {
 
         public BooleanFieldType(
             String name,
-            boolean isIndexed,
+            IndexType indexType,
             boolean isStored,
-            boolean hasDocValues,
             Boolean nullValue,
             FieldValues<Boolean> scriptValues,
             Map<String, String> meta,
             boolean isDimension,
             boolean isSyntheticSource
         ) {
-            super(name, IndexType.terms(isIndexed, hasDocValues), isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
+            super(name, indexType, isStored, TextSearchInfo.SIMPLE_MATCH_ONLY, meta);
             this.nullValue = nullValue;
             this.scriptValues = scriptValues;
             this.isDimension = isDimension;
             this.isSyntheticSource = isSyntheticSource;
         }
 
         public BooleanFieldType(String name) {
-            this(name, true);
-        }
-
-        public BooleanFieldType(String name, boolean isIndexed) {
-            this(name, isIndexed, true);
+            this(name, IndexType.terms(true, true));
         }
 
-        public BooleanFieldType(String name, boolean isIndexed, boolean hasDocValues) {
-            this(name, isIndexed, isIndexed, hasDocValues, false, null, Collections.emptyMap(), false, false);
+        public BooleanFieldType(String name, IndexType indexType) {
+            this(name, indexType, true, false, null, Collections.emptyMap(), false, false);
         }
 
         @Override
@@ -601,7 +599,11 @@ private void indexValue(DocumentParserContext context, Boolean value) {
             context.doc().add(new StoredField(fieldType().name(), value ? "T" : "F"));
         }
         if (hasDocValues) {
-            context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0));
+            if (fieldType().indexType.hasDocValuesSkipper()) {
+                context.doc().add(SortedNumericDocValuesField.indexedField(fieldType().name(), value ? 1 : 0));
+            } else {
+                context.doc().add(new SortedNumericDocValuesField(fieldType().name(), value ? 1 : 0));
+            }
         } else {
             context.addToFieldNames(fieldType().name());
         }

server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java

@@ -1314,6 +1314,21 @@ public static Parameter<Boolean> indexParam(Function<FieldMapper, Boolean> initi
             return Parameter.boolParam("index", false, initializer, defaultValue);
         }
 
+        public static Parameter<Boolean> indexParam(
+            Function<FieldMapper, Boolean> initializer,
+            IndexSettings indexSettings,
+            Supplier<Boolean> isDimension
+        ) {
+            return Parameter.boolParam(
+                "index",
+                false,
+                initializer,
+                () -> isDimension.get() == false
+                    || indexSettings.useDocValuesSkipper() == false
+                    || indexSettings.getIndexVersionCreated().before(IndexVersions.TIME_SERIES_DIMENSIONS_USE_SKIPPERS)
+            );
+        }
+
         public static Parameter<Boolean> storeParam(Function<FieldMapper, Boolean> initializer, boolean defaultValue) {
             return Parameter.boolParam("store", false, initializer, defaultValue);
         }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -143,7 +143,7 @@ public Builder(String name, ScriptCompiler scriptCompiler, IndexSettings indexSe
                 }
             });
             // We allow `time_series_dimension` parameter to be parsed, but only allow it to be `false`
-            this.dimension = TimeSeriesParams.dimensionParam(m -> false).addValidator(v -> {
+            this.dimension = TimeSeriesParams.dimensionParam(m -> false, () -> true).addValidator(v -> {
                 if (v) {
                     throw new IllegalArgumentException(
                         "Parameter [" + TimeSeriesParams.TIME_SERIES_DIMENSION_PARAM + "] cannot be set to geo_point"

server/src/main/java/org/elasticsearch/index/mapper/IndexType.java

@@ -9,6 +9,11 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.DocValuesSkipIndexType;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
+
 import java.util.Objects;
 
 /**
@@ -118,6 +123,25 @@ public static IndexType terms(boolean isIndexed, boolean hasDocValues) {
         return new IndexType(isIndexed, false, false, false, hasDocValues, false);
     }
 
+    /**
+     * @return a terms-based IndexType from a lucene FieldType
+     */
+    public static IndexType terms(FieldType fieldType) {
+        if (fieldType.indexOptions() == IndexOptions.NONE) {
+            if (fieldType.docValuesType() == DocValuesType.NONE) {
+                return NONE;
+            }
+            if (fieldType.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) {
+                return docValuesOnly();
+            }
+            return skippers();
+        }
+        if (fieldType.docValuesType() == DocValuesType.NONE) {
+            return terms(true, false);
+        }
+        return terms(true, true);
+    }
+
     /**
      * @return an IndexType with docValuesSkippers
      */