Add index.use_time_series_doc_values_format index setting.

docs/reference/elasticsearch/index-settings/index-modules.md

@@ -154,7 +154,7 @@ $$$index-refresh-interval-setting$$$
 :   How often to perform a refresh operation, which makes recent changes to the index visible to search. If this setting is not explicitly set, shards that haven’t seen search traffic for at least `index.search.idle.after` seconds will not receive background refreshes until they receive a search request. Searches that hit an idle shard where a refresh is pending will trigger a refresh as part of the search operation for that shard only. This behavior aims to automatically optimize bulk indexing in the default case when no searches are performed. To opt out of this behavior, set an explicit value for the refresh interval, even if it matches the default value.
 
     The value defaults to `1s` in {{stack}} and `5s` in {{serverless-short}}. In {{serverless-short}}, `5s` is also the minimum value that can be set.
-    
+
     In both cases, the setting can be set to `-1` to disable refresh.
 
 $$$index-max-result-window$$$
@@ -272,3 +272,6 @@ $$$index-esql-stored-fields-sequential-proportion$$$
 
 $$$index-dense-vector-hnsw-early-termination$$$ `index.dense_vector.hnsw_early_termination` {applies_to}`stack: ga 9.2` {applies_to}`serverless: all`
 :   Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`.
+
+$$$index-use_time_series_doc_values_format$$$ `index.use_time_series_doc_values_format` {applies_to}`stack: ga 9.3`
+:   Indicates whether the time series doc values format should be used. Defaults to `true` if `index.mode` is `time_series` or `logsdb`, otherwise `false`.

server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java

@@ -207,6 +207,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING,
                 IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING,
                 IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING,
+                IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING,
                 InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT,
 
                 // validate that built-in similarities don't get redefined

server/src/main/java/org/elasticsearch/index/IndexMode.java

@@ -246,6 +246,11 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) {
         public SourceFieldMapper.Mode defaultSourceMode() {
             return SourceFieldMapper.Mode.SYNTHETIC;
         }
+
+        @Override
+        public boolean useTimeSeriesDocValuesCodec() {
+            return true;
+        }
     },
     LOGSDB("logsdb") {
         @Override
@@ -331,6 +336,11 @@ public SourceFieldMapper.Mode defaultSourceMode() {
         public String getDefaultCodec() {
             return CodecService.BEST_COMPRESSION_CODEC;
         }
+
+        @Override
+        public boolean useTimeSeriesDocValuesCodec() {
+            return true;
+        }
     },
     LOOKUP("lookup") {
         @Override
@@ -571,6 +581,13 @@ public boolean useDefaultPostingsFormat() {
         return false;
     }
 
+    /**
+     * Whether by default to use the time series doc values codec.
+     */
+    public boolean useTimeSeriesDocValuesCodec() {
+        return false;
+    }
+
     /**
      * Parse a string into an {@link IndexMode}.
      */

server/src/main/java/org/elasticsearch/index/IndexSettings.java

@@ -631,6 +631,7 @@ public Iterator<Setting<?>> settings() {
         Property.ServerlessPublic
     );
 
+    // TODO: deprecate this setting as this was designed as an opt-out and name of the setting is tied actual name of the codec:
     public static final Setting<Boolean> TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING = Setting.boolSetting(
         "index.time_series.es87tsdb_codec.enabled",
         true,
@@ -827,6 +828,19 @@ public Iterator<Setting<?>> settings() {
         Property.Final
     );
 
+    public static final Setting<Boolean> USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING = Setting.boolSetting(
+        "index.use_time_series_doc_values_format",
+        settings -> {
+            if (settings == null) {
+                return Boolean.FALSE.toString();
+            }
+            var indexMode = IndexSettings.MODE.get(settings);
+            return Boolean.toString(indexMode.useTimeSeriesDocValuesCodec());
+        },
+        Property.IndexScope,
+        Property.Final
+    );
+
     /**
      * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use.
      * TODO: Remove in 9.0
@@ -996,6 +1010,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) {
     private final boolean recoverySourceSyntheticEnabled;
     private final boolean useDocValuesSkipper;
     private final boolean useTimeSeriesSyntheticId;
+    private final boolean useTimeSeriesDocValuesFormat;
 
     /**
      * The maximum number of refresh listeners allows on this shard.
@@ -1182,6 +1197,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
             && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING);
         useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER);
         seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING);
+        useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING);
         final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID);
         if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) {
             assert false;
@@ -1943,6 +1959,13 @@ public boolean useTimeSeriesSyntheticId() {
         return useTimeSeriesSyntheticId;
     }
 
+    /**
+     * @return Whether the time series doc value format should be used.
+     */
+    public boolean useTimeSeriesDocValuesFormat() {
+        return useTimeSeriesDocValuesFormat;
+    }
+
     /**
      * The bounds for {@code @timestamp} on this index or
      * {@code null} if there are no bounds.

server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java

@@ -139,20 +139,12 @@ boolean useTSDBDocValuesFormat(final String field) {
         }
 
         return mapperService != null
-            && (isTimeSeriesModeIndex() || isLogsModeIndex())
+            && mapperService.getIndexSettings().useTimeSeriesDocValuesFormat()
             && mapperService.getIndexSettings().isES87TSDBCodecEnabled();
     }
 
     private boolean excludeFields(String fieldName) {
         return fieldName.startsWith("_") && INCLUDE_META_FIELDS.contains(fieldName) == false;
     }
 
-    private boolean isTimeSeriesModeIndex() {
-        return mapperService != null && IndexMode.TIME_SERIES == mapperService.getIndexSettings().getMode();
-    }
-
-    private boolean isLogsModeIndex() {
-        return mapperService != null && IndexMode.LOGSDB == mapperService.getIndexSettings().getMode();
-    }
-
 }

server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java

@@ -185,12 +185,19 @@ public void testUseES87TSDBEncodingSettingDisabled() throws IOException {
     }
 
     public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.STANDARD, MAPPING_2);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.STANDARD, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false));
     }
 
+    public void testUseTimeSeriesDocValuesCodecSetting() throws IOException {
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, null, IndexMode.STANDARD, MAPPING_2);
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true));
+        assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true));
+    }
+
     public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
         PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
@@ -199,14 +206,14 @@ public void testUseTimeSeriesModeAndCodecEnabled() throws IOException {
     }
 
     public void testLogsIndexMode() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true));
     }
 
     public void testMetaFields() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesIdFieldMapper.NAME)), is(true));
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesRoutingHashFieldMapper.NAME)), is(true));
         // See: PerFieldFormatSupplier why these fields shouldn't use tsdb codec
@@ -215,17 +222,35 @@ public void testMetaFields() throws IOException {
     }
 
     public void testSeqnoField() throws IOException {
-        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3);
+        PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3);
         assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SeqNoFieldMapper.NAME)), is(true));
     }
 
-    private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
+    private PerFieldFormatSupplier createFormatSupplier(IndexMode mode, String mapping) throws IOException {
+        return createFormatSupplier(null, mode, mapping);
+    }
+
+    private PerFieldFormatSupplier createFormatSupplier(Boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException {
+        return createFormatSupplier(null, enableES87TSDBCodec, mode, mapping);
+    }
+
+    private PerFieldFormatSupplier createFormatSupplier(
+        Boolean useTimeSeriesDocValuesFormatSetting,
+        Boolean enableES87TSDBCodec,
+        IndexMode mode,
+        String mapping
+    ) throws IOException {
         Settings.Builder settings = Settings.builder();
         settings.put(IndexSettings.MODE.getKey(), mode);
         if (mode == IndexMode.TIME_SERIES) {
             settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field");
         }
-        settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec);
+        if (enableES87TSDBCodec != null) {
+            settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec);
+        }
+        if (useTimeSeriesDocValuesFormatSetting != null) {
+            settings.put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), useTimeSeriesDocValuesFormatSetting);
+        }
         MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test");
         mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE);
         return new PerFieldFormatSupplier(mapperService, BigArrays.NON_RECYCLING_INSTANCE);