From 01d89af0ef1bf22080ceee04271de4bee915c7e1 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 13 Mar 2025 20:12:41 +0100 Subject: [PATCH] Revert "Sparse doc values index for LogsDB `host.name` field (#120741)" This reverts commit 1b6a080473f1a7a182f08e2ef322d6b9ef30cee8. --- .../index/mapper/FieldMapper.java | 6 - .../index/mapper/KeywordFieldMapper.java | 143 +-------------- .../index/mapper/KeywordFieldMapperTests.java | 172 ------------------ 3 files changed, 9 insertions(+), 312 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 2dfbf871ea183..7c1f3678a5dc9 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -21,7 +21,6 @@ import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Setting.Property; import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.util.FeatureFlag; import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.support.XContentMapValues; import org.elasticsearch.index.IndexMode; @@ -64,7 +63,6 @@ public abstract class FieldMapper extends Mapper { private static final Logger logger = LogManager.getLogger(FieldMapper.class); - public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index"); public static final Setting IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> { if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) { @@ -853,10 +851,6 @@ public boolean isConfigured() { return isSet && Objects.equals(value, getDefaultValue()) == false; } - public boolean isSet() { - return isSet; - } - /** * Allows the parameter to accept a {@code null} value */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b7528bd3729ee..7b14739d36246 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -18,7 +18,6 @@ import org.apache.lucene.document.InvertableType; import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexOptions; import org.apache.lucene.index.IndexReader; @@ -39,10 +38,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -91,11 +87,9 @@ public final class KeywordFieldMapper extends FieldMapper { private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class); public static final String CONTENT_TYPE = "keyword"; - private static final String HOST_NAME = "host.name"; public static class Defaults { public static final FieldType FIELD_TYPE; - public static final FieldType FIELD_TYPE_WITH_SKIP_DOC_VALUES; static { FieldType ft = new FieldType(); @@ -106,16 +100,6 @@ public static class Defaults { FIELD_TYPE = freezeAndDeduplicateFieldType(ft); } - static { - FieldType ft = new FieldType(); - ft.setTokenized(false); - ft.setOmitNorms(true); - ft.setIndexOptions(IndexOptions.NONE); - ft.setDocValuesType(DocValuesType.SORTED_SET); - ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE); - FIELD_TYPE_WITH_SKIP_DOC_VALUES = freezeAndDeduplicateFieldType(ft); - } - public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( FIELD_TYPE, null, @@ -170,8 +154,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { ); private final Parameter ignoreAbove; private final int ignoreAboveDefault; - private final IndexSortConfig indexSortConfig; - private final IndexMode indexMode; + private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); private final Parameter hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false); private final Parameter similarity = TextParams.similarity( @@ -206,9 +189,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), - mappingParserContext.getIndexSettings().getIndexVersionCreated(), - mappingParserContext.getIndexSettings().getMode(), - mappingParserContext.getIndexSettings().getIndexSortConfig() + mappingParserContext.getIndexSettings().getIndexVersionCreated() ); } @@ -218,18 +199,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex ScriptCompiler scriptCompiler, int ignoreAboveDefault, IndexVersion indexCreatedVersion - ) { - this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null); - } - - private Builder( - String name, - IndexAnalyzers indexAnalyzers, - ScriptCompiler scriptCompiler, - int ignoreAboveDefault, - IndexVersion indexCreatedVersion, - IndexMode indexMode, - IndexSortConfig indexSortConfig ) { super(name); this.indexAnalyzers = indexAnalyzers; @@ -264,8 +233,6 @@ private Builder( throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); } }); - this.indexSortConfig = indexSortConfig; - this.indexMode = indexMode; } public Builder(String name, IndexVersion indexCreatedVersion) { @@ -392,21 +359,15 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { - FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName())); + FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE); fieldtype.setOmitNorms(this.hasNorms.getValue() == false); + fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) { - // NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index) - fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); - } if (fieldtype.equals(Defaults.FIELD_TYPE)) { // deduplicate in the common default case to save some memory fieldtype = Defaults.FIELD_TYPE; } - if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) { - fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES; - } super.hasScript = script.get() != null; super.onScriptError = onScriptError.getValue(); return new KeywordFieldMapper( @@ -418,63 +379,6 @@ public KeywordFieldMapper build(MapperBuilderContext context) { this ); } - - private FieldType resolveFieldType( - final IndexVersion indexCreatedVersion, - final IndexSortConfig indexSortConfig, - final IndexMode indexMode, - final String fullFieldName - ) { - if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled() - && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) - && shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) { - return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); - } - return new FieldType(Defaults.FIELD_TYPE); - } - - /** - * Determines whether to use a sparse index representation for doc values. - * - *

If the field is explicitly indexed by setting {@code index: true}, we do not use - * a sparse doc values index but instead rely on the inverted index, as is typically - * the case for keyword fields.

- * - *

This method checks several conditions to decide if the sparse index format - * should be applied:

- * - *
    - *
  • Returns {@code false} immediately if the field is explicitly indexed.
    • - *
  • Ensures the field is not explicitly configured as indexed (i.e., {@code index} has its default value).
    • - *
  • Requires doc values to be enabled.
    • - *
  • Index mode must be {@link IndexMode#LOGSDB}.
    • - *
  • Field name must be {@code host.name}.
    • - *
  • The {@code host.name} field must be a primary sort field.
    • - *
- * - *

Returns {@code true} if all conditions are met, indicating that sparse doc values - * should be used. Otherwise, returns {@code false}.

- * - * @param indexSortConfig The index sort configuration, used to check primary sorting. - * @param indexMode The mode of the index, which must be {@link IndexMode#LOGSDB}. - * @param fullFieldName The name of the field being checked, which must be {@code host.name}. - * @return {@code true} if sparse doc values should be used, otherwise {@code false}. - */ - - private boolean shouldUseDocValuesSparseIndex( - final IndexSortConfig indexSortConfig, - final IndexMode indexMode, - final String fullFieldName - ) { - if (indexed.isSet() && indexed.getValue()) { - return false; - } - return indexed.isConfigured() == false - && hasDocValues.getValue() - && IndexMode.LOGSDB.equals(indexMode) - && HOST_NAME.equals(fullFieldName) - && (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME)); - } } public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); @@ -488,9 +392,6 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; - private final IndexMode indexMode; - private final IndexSortConfig indexSortConfig; - private final boolean hasDocValuesSparseIndex; public KeywordFieldType( String name, @@ -516,9 +417,6 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; - this.indexMode = builder.indexMode; - this.indexSortConfig = builder.indexSortConfig; - this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -530,9 +428,6 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; - this.indexSortConfig = null; - this.hasDocValuesSparseIndex = false; } public KeywordFieldType(String name) { @@ -555,9 +450,6 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; - this.indexSortConfig = null; - this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -569,9 +461,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; - this.indexSortConfig = null; - this.hasDocValuesSparseIndex = false; } @Override @@ -962,18 +851,6 @@ public boolean hasScriptValues() { public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } - - public IndexMode getIndexMode() { - return indexMode; - } - - public IndexSortConfig getIndexSortConfig() { - return indexSortConfig; - } - - public boolean hasDocValuesSparseIndex() { - return hasDocValuesSparseIndex; - } } private final boolean indexed; @@ -989,8 +866,7 @@ public boolean hasDocValuesSparseIndex() { private final IndexAnalyzers indexAnalyzers; private final int ignoreAboveDefault; - private final IndexMode indexMode; - private final IndexSortConfig indexSortConfig; + private final int ignoreAbove; private KeywordFieldMapper( String simpleName, @@ -1014,8 +890,7 @@ private KeywordFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; this.ignoreAboveDefault = builder.ignoreAboveDefault; - this.indexMode = builder.indexMode; - this.indexSortConfig = builder.indexSortConfig; + this.ignoreAbove = builder.ignoreAbove.getValue(); } @Override @@ -1133,9 +1008,9 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig) - .dimension(fieldType().isDimension()) - .init(this); + return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension( + fieldType().isDimension() + ).init(this); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index 924c32f527f7a..052bf995bdd48 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -28,7 +28,6 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -771,175 +770,4 @@ public void testDocValuesLoadedFromStoredSynthetic() throws IOException { ); assertScriptDocValues(mapper, "foo", equalTo(List.of("foo"))); } - - public void testFieldTypeWithSkipDocValues_LogsDbMode() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_StandardMode() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_NonMatchingFieldName() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "hostname") - .build(), - mapping(b -> { - b.startObject("hostname"); - b.field("type", "keyword"); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("hostname"); - assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_ConfiguredIndexed() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_ConfiguredDocValues() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("doc_values", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertTrue(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_LogsDbMode_NonSortField() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder().put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()).build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertTrue(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeWithSkipDocValues_IndexedFalseDocValuesTrue() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", false); - b.field("doc_values", true); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertTrue(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } - - public void testFieldTypeDefault_IndexedFalseDocValuesFalse() throws IOException { - assumeTrue("Needs feature flag to be enabled", FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()); - - final MapperService mapperService = createMapperService( - Settings.builder() - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) - .put(IndexSortConfig.INDEX_SORT_FIELD_SETTING.getKey(), "host.name") - .build(), - mapping(b -> { - b.startObject("host.name"); - b.field("type", "keyword"); - b.field("index", false); - b.field("doc_values", false); - b.endObject(); - }) - ); - - final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("host.name"); - assertFalse(mapper.fieldType().hasDocValues()); - assertFalse(mapper.fieldType().isIndexed()); - assertFalse(mapper.fieldType().hasDocValuesSparseIndex()); - } }