diff --git a/docs/reference/elasticsearch/index-settings/index-modules.md b/docs/reference/elasticsearch/index-settings/index-modules.md index 708578b023c1a..7c376db35aba9 100644 --- a/docs/reference/elasticsearch/index-settings/index-modules.md +++ b/docs/reference/elasticsearch/index-settings/index-modules.md @@ -154,7 +154,7 @@ $$$index-refresh-interval-setting$$$ : How often to perform a refresh operation, which makes recent changes to the index visible to search. If this setting is not explicitly set, shards that haven’t seen search traffic for at least `index.search.idle.after` seconds will not receive background refreshes until they receive a search request. Searches that hit an idle shard where a refresh is pending will trigger a refresh as part of the search operation for that shard only. This behavior aims to automatically optimize bulk indexing in the default case when no searches are performed. To opt out of this behavior, set an explicit value for the refresh interval, even if it matches the default value. The value defaults to `1s` in {{stack}} and `5s` in {{serverless-short}}. In {{serverless-short}}, `5s` is also the minimum value that can be set. - + In both cases, the setting can be set to `-1` to disable refresh. $$$index-max-result-window$$$ @@ -272,3 +272,6 @@ $$$index-esql-stored-fields-sequential-proportion$$$ $$$index-dense-vector-hnsw-early-termination$$$ `index.dense_vector.hnsw_early_termination` {applies_to}`stack: ga 9.2` {applies_to}`serverless: all` : Whether to apply _patience_ based early termination strategy to knn queries over HNSW graphs (see [paper](https://cs.uwaterloo.ca/~jimmylin/publications/Teofili_Lin_ECIR2025.pdf)). This is only applicable to `dense_vector` fields with `hnsw`, `int8_hnsw`, `int4_hnsw` and `bbq_hnsw` index types. Defaults to `false`. + +$$$index-use_time_series_doc_values_format$$$ `index.use_time_series_doc_values_format` {applies_to}`stack: ga 9.3` +: Indicates whether the time series doc values format should be used. Defaults to `true` if `index.mode` is `time_series` or `logsdb`, otherwise `false`. diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 9f2c6c4d17355..023c532798082 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -207,6 +207,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IgnoredSourceFieldMapper.SKIP_IGNORED_SOURCE_READ_SETTING, IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING, IndexSettings.RECOVERY_USE_SYNTHETIC_SOURCE_SETTING, + IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING, InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT, // validate that built-in similarities don't get redefined diff --git a/server/src/main/java/org/elasticsearch/index/IndexMode.java b/server/src/main/java/org/elasticsearch/index/IndexMode.java index 10e604126f934..1a8916bbd8875 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexMode.java +++ b/server/src/main/java/org/elasticsearch/index/IndexMode.java @@ -246,6 +246,11 @@ public void validateSourceFieldMapper(SourceFieldMapper sourceFieldMapper) { public SourceFieldMapper.Mode defaultSourceMode() { return SourceFieldMapper.Mode.SYNTHETIC; } + + @Override + public boolean useTimeSeriesDocValuesCodec() { + return true; + } }, LOGSDB("logsdb") { @Override @@ -331,6 +336,11 @@ public SourceFieldMapper.Mode defaultSourceMode() { public String getDefaultCodec() { return CodecService.BEST_COMPRESSION_CODEC; } + + @Override + public boolean useTimeSeriesDocValuesCodec() { + return true; + } }, LOOKUP("lookup") { @Override @@ -571,6 +581,13 @@ public boolean useDefaultPostingsFormat() { return false; } + /** + * Whether by default to use the time series doc values codec. + */ + public boolean useTimeSeriesDocValuesCodec() { + return false; + } + /** * Parse a string into an {@link IndexMode}. */ diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 81b4bbab69756..5b1d86fe415eb 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -631,6 +631,7 @@ public Iterator> settings() { Property.ServerlessPublic ); + // TODO: deprecate this setting as this was designed as an opt-out and name of the setting is tied actual name of the codec: public static final Setting TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING = Setting.boolSetting( "index.time_series.es87tsdb_codec.enabled", true, @@ -827,6 +828,19 @@ public Iterator> settings() { Property.Final ); + public static final Setting USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING = Setting.boolSetting( + "index.use_time_series_doc_values_format", + settings -> { + if (settings == null) { + return Boolean.FALSE.toString(); + } + var indexMode = IndexSettings.MODE.get(settings); + return Boolean.toString(indexMode.useTimeSeriesDocValuesCodec()); + }, + Property.IndexScope, + Property.Final + ); + /** * Legacy index setting, kept for 7.x BWC compatibility. This setting has no effect in 8.x. Do not use. * TODO: Remove in 9.0 @@ -996,6 +1010,7 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private final boolean recoverySourceSyntheticEnabled; private final boolean useDocValuesSkipper; private final boolean useTimeSeriesSyntheticId; + private final boolean useTimeSeriesDocValuesFormat; /** * The maximum number of refresh listeners allows on this shard. @@ -1182,6 +1197,7 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti && scopedSettings.get(RECOVERY_USE_SYNTHETIC_SOURCE_SETTING); useDocValuesSkipper = DOC_VALUES_SKIPPER && scopedSettings.get(USE_DOC_VALUES_SKIPPER); seqNoIndexOptions = scopedSettings.get(SEQ_NO_INDEX_OPTIONS_SETTING); + useTimeSeriesDocValuesFormat = scopedSettings.get(USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING); final var useSyntheticId = IndexSettings.TSDB_SYNTHETIC_ID_FEATURE_FLAG && scopedSettings.get(USE_SYNTHETIC_ID); if (indexMetadata.useTimeSeriesSyntheticId() != useSyntheticId) { assert false; @@ -1943,6 +1959,13 @@ public boolean useTimeSeriesSyntheticId() { return useTimeSeriesSyntheticId; } + /** + * @return Whether the time series doc value format should be used. + */ + public boolean useTimeSeriesDocValuesFormat() { + return useTimeSeriesDocValuesFormat; + } + /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java index 2ed1aa6c9f17f..f070139872f98 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldFormatSupplier.java @@ -139,7 +139,7 @@ boolean useTSDBDocValuesFormat(final String field) { } return mapperService != null - && (isTimeSeriesModeIndex() || isLogsModeIndex()) + && mapperService.getIndexSettings().useTimeSeriesDocValuesFormat() && mapperService.getIndexSettings().isES87TSDBCodecEnabled(); } @@ -147,12 +147,4 @@ private boolean excludeFields(String fieldName) { return fieldName.startsWith("_") && INCLUDE_META_FIELDS.contains(fieldName) == false; } - private boolean isTimeSeriesModeIndex() { - return mapperService != null && IndexMode.TIME_SERIES == mapperService.getIndexSettings().getMode(); - } - - private boolean isLogsModeIndex() { - return mapperService != null && IndexMode.LOGSDB == mapperService.getIndexSettings().getMode(); - } - } diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index 6c17c70f8dcf1..535ad009c513e 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -185,12 +185,19 @@ public void testUseES87TSDBEncodingSettingDisabled() throws IOException { } public void testUseTimeSeriesModeDisabledCodecDisabled() throws IOException { - PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.STANDARD, MAPPING_2); + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.STANDARD, MAPPING_2); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(false)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(false)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(false)); } + public void testUseTimeSeriesDocValuesCodecSetting() throws IOException { + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, null, IndexMode.STANDARD, MAPPING_2); + assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true)); + assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("counter")), is(true)); + assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("gauge")), is(true)); + } + public void testUseTimeSeriesModeAndCodecEnabled() throws IOException { PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.TIME_SERIES, MAPPING_2); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true)); @@ -199,14 +206,14 @@ public void testUseTimeSeriesModeAndCodecEnabled() throws IOException { } public void testLogsIndexMode() throws IOException { - PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3); + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("@timestamp")), is(true)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("hostname")), is(true)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat("response_size")), is(true)); } public void testMetaFields() throws IOException { - PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3); + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesIdFieldMapper.NAME)), is(true)); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(TimeSeriesRoutingHashFieldMapper.NAME)), is(true)); // See: PerFieldFormatSupplier why these fields shouldn't use tsdb codec @@ -215,17 +222,35 @@ public void testMetaFields() throws IOException { } public void testSeqnoField() throws IOException { - PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(true, IndexMode.LOGSDB, MAPPING_3); + PerFieldFormatSupplier perFieldMapperCodec = createFormatSupplier(IndexMode.LOGSDB, MAPPING_3); assertThat((perFieldMapperCodec.useTSDBDocValuesFormat(SeqNoFieldMapper.NAME)), is(true)); } - private PerFieldFormatSupplier createFormatSupplier(boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException { + private PerFieldFormatSupplier createFormatSupplier(IndexMode mode, String mapping) throws IOException { + return createFormatSupplier(null, mode, mapping); + } + + private PerFieldFormatSupplier createFormatSupplier(Boolean enableES87TSDBCodec, IndexMode mode, String mapping) throws IOException { + return createFormatSupplier(null, enableES87TSDBCodec, mode, mapping); + } + + private PerFieldFormatSupplier createFormatSupplier( + Boolean useTimeSeriesDocValuesFormatSetting, + Boolean enableES87TSDBCodec, + IndexMode mode, + String mapping + ) throws IOException { Settings.Builder settings = Settings.builder(); settings.put(IndexSettings.MODE.getKey(), mode); if (mode == IndexMode.TIME_SERIES) { settings.put(IndexMetadata.INDEX_ROUTING_PATH.getKey(), "field"); } - settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec); + if (enableES87TSDBCodec != null) { + settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec); + } + if (useTimeSeriesDocValuesFormatSetting != null) { + settings.put(IndexSettings.USE_TIME_SERIES_DOC_VALUES_FORMAT_SETTING.getKey(), useTimeSeriesDocValuesFormatSetting); + } MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test"); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); return new PerFieldFormatSupplier(mapperService, BigArrays.NON_RECYCLING_INSTANCE);