Skip to content

Remove doc value skipper experimental code from 9.0 branch. #124803

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 13, 2025
Merged

Remove doc value skipper experimental code from 9.0 branch. #124803

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.util.CollectionUtils;
import org.elasticsearch.common.util.FeatureFlag;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.support.XContentMapValues;
import org.elasticsearch.index.IndexMode;
Expand Down Expand Up @@ -64,7 +63,6 @@
public abstract class FieldMapper extends Mapper {
private static final Logger logger = LogManager.getLogger(FieldMapper.class);

public static final FeatureFlag DOC_VALUES_SPARSE_INDEX = new FeatureFlag("doc_values_sparse_index");
public static final Setting<Boolean> IGNORE_MALFORMED_SETTING = Setting.boolSetting("index.mapping.ignore_malformed", settings -> {
if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB
&& IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB)) {
Expand Down Expand Up @@ -853,10 +851,6 @@ public boolean isConfigured() {
return isSet && Objects.equals(value, getDefaultValue()) == false;
}

public boolean isSet() {
return isSet;
}

/**
* Allows the parameter to accept a {@code null} value
*/
Expand Down
143 changes: 9 additions & 134 deletions server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
import org.apache.lucene.document.InvertableType;
import org.apache.lucene.document.SortedSetDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DocValuesSkipIndexType;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
Expand All @@ -39,10 +38,7 @@
import org.elasticsearch.common.lucene.search.AutomatonQueries;
import org.elasticsearch.common.unit.Fuzziness;
import org.elasticsearch.core.Nullable;
import org.elasticsearch.index.IndexMode;
import org.elasticsearch.index.IndexSortConfig;
import org.elasticsearch.index.IndexVersion;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.analysis.IndexAnalyzers;
import org.elasticsearch.index.analysis.NamedAnalyzer;
import org.elasticsearch.index.fielddata.FieldData;
Expand Down Expand Up @@ -91,11 +87,9 @@ public final class KeywordFieldMapper extends FieldMapper {
private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class);

public static final String CONTENT_TYPE = "keyword";
private static final String HOST_NAME = "host.name";

public static class Defaults {
public static final FieldType FIELD_TYPE;
public static final FieldType FIELD_TYPE_WITH_SKIP_DOC_VALUES;

static {
FieldType ft = new FieldType();
Expand All @@ -106,16 +100,6 @@ public static class Defaults {
FIELD_TYPE = freezeAndDeduplicateFieldType(ft);
}

static {
FieldType ft = new FieldType();
ft.setTokenized(false);
ft.setOmitNorms(true);
ft.setIndexOptions(IndexOptions.NONE);
ft.setDocValuesType(DocValuesType.SORTED_SET);
ft.setDocValuesSkipIndexType(DocValuesSkipIndexType.RANGE);
FIELD_TYPE_WITH_SKIP_DOC_VALUES = freezeAndDeduplicateFieldType(ft);
}

public static final TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo(
FIELD_TYPE,
null,
Expand Down Expand Up @@ -170,8 +154,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder {
);
private final Parameter<Integer> ignoreAbove;
private final int ignoreAboveDefault;
private final IndexSortConfig indexSortConfig;
private final IndexMode indexMode;

private final Parameter<String> indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions);
private final Parameter<Boolean> hasNorms = TextParams.norms(false, m -> toType(m).fieldType.omitNorms() == false);
private final Parameter<SimilarityProvider> similarity = TextParams.similarity(
Expand Down Expand Up @@ -206,9 +189,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex
mappingParserContext.getIndexAnalyzers(),
mappingParserContext.scriptCompiler(),
IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()),
mappingParserContext.getIndexSettings().getIndexVersionCreated(),
mappingParserContext.getIndexSettings().getMode(),
mappingParserContext.getIndexSettings().getIndexSortConfig()
mappingParserContext.getIndexSettings().getIndexVersionCreated()
);
}

Expand All @@ -218,18 +199,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion
) {
this(name, indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, IndexMode.STANDARD, null);
}

private Builder(
String name,
IndexAnalyzers indexAnalyzers,
ScriptCompiler scriptCompiler,
int ignoreAboveDefault,
IndexVersion indexCreatedVersion,
IndexMode indexMode,
IndexSortConfig indexSortConfig
) {
super(name);
this.indexAnalyzers = indexAnalyzers;
Expand Down Expand Up @@ -264,8 +233,6 @@ private Builder(
throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]");
}
});
this.indexSortConfig = indexSortConfig;
this.indexMode = indexMode;
}

public Builder(String name, IndexVersion indexCreatedVersion) {
Expand Down Expand Up @@ -392,21 +359,15 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType

@Override
public KeywordFieldMapper build(MapperBuilderContext context) {
FieldType fieldtype = resolveFieldType(indexCreatedVersion, indexSortConfig, indexMode, context.buildFullName(leafName()));
FieldType fieldtype = new FieldType(Defaults.FIELD_TYPE);
fieldtype.setOmitNorms(this.hasNorms.getValue() == false);
fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue()));
fieldtype.setStored(this.stored.getValue());
fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE);
if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES) == false) {
// NOTE: override index options only if we are not using a sparse doc values index (and we use an inverted index)
fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue()));
}
if (fieldtype.equals(Defaults.FIELD_TYPE)) {
// deduplicate in the common default case to save some memory
fieldtype = Defaults.FIELD_TYPE;
}
if (fieldtype.equals(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES)) {
fieldtype = Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES;
}
super.hasScript = script.get() != null;
super.onScriptError = onScriptError.getValue();
return new KeywordFieldMapper(
Expand All @@ -418,63 +379,6 @@ public KeywordFieldMapper build(MapperBuilderContext context) {
this
);
}

private FieldType resolveFieldType(
final IndexVersion indexCreatedVersion,
final IndexSortConfig indexSortConfig,
final IndexMode indexMode,
final String fullFieldName
) {
if (FieldMapper.DOC_VALUES_SPARSE_INDEX.isEnabled()
&& indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX)
&& shouldUseDocValuesSparseIndex(indexSortConfig, indexMode, fullFieldName)) {
return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES);
}
return new FieldType(Defaults.FIELD_TYPE);
}

/**
* Determines whether to use a sparse index representation for doc values.
*
* <p>If the field is explicitly indexed by setting {@code index: true}, we do not use
* a sparse doc values index but instead rely on the inverted index, as is typically
* the case for keyword fields.</p>
*
* <p>This method checks several conditions to decide if the sparse index format
* should be applied:</p>
*
* <ul>
* <li>Returns {@code false} immediately if the field is explicitly indexed.</li>
* <li>Ensures the field is not explicitly configured as indexed (i.e., {@code index} has its default value).</li>
* <li>Requires doc values to be enabled.</li>
* <li>Index mode must be {@link IndexMode#LOGSDB}.</li>
* <li>Field name must be {@code host.name}.</li>
* <li>The {@code host.name} field must be a primary sort field.</li>
* </ul>
*
* <p>Returns {@code true} if all conditions are met, indicating that sparse doc values
* should be used. Otherwise, returns {@code false}.</p>
*
* @param indexSortConfig The index sort configuration, used to check primary sorting.
* @param indexMode The mode of the index, which must be {@link IndexMode#LOGSDB}.
* @param fullFieldName The name of the field being checked, which must be {@code host.name}.
* @return {@code true} if sparse doc values should be used, otherwise {@code false}.
*/

private boolean shouldUseDocValuesSparseIndex(
final IndexSortConfig indexSortConfig,
final IndexMode indexMode,
final String fullFieldName
) {
if (indexed.isSet() && indexed.getValue()) {
return false;
}
return indexed.isConfigured() == false
&& hasDocValues.getValue()
&& IndexMode.LOGSDB.equals(indexMode)
&& HOST_NAME.equals(fullFieldName)
&& (indexSortConfig != null && indexSortConfig.hasPrimarySortOnField(HOST_NAME));
}
}

public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new);
Expand All @@ -488,9 +392,6 @@ public static final class KeywordFieldType extends StringFieldType {
private final FieldValues<String> scriptValues;
private final boolean isDimension;
private final boolean isSyntheticSource;
private final IndexMode indexMode;
private final IndexSortConfig indexSortConfig;
private final boolean hasDocValuesSparseIndex;

public KeywordFieldType(
String name,
Expand All @@ -516,9 +417,6 @@ public KeywordFieldType(
this.scriptValues = builder.scriptValues();
this.isDimension = builder.dimension.getValue();
this.isSyntheticSource = isSyntheticSource;
this.indexMode = builder.indexMode;
this.indexSortConfig = builder.indexSortConfig;
this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
}

public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map<String, String> meta) {
Expand All @@ -530,9 +428,6 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = false;
}

public KeywordFieldType(String name) {
Expand All @@ -555,9 +450,6 @@ public KeywordFieldType(String name, FieldType fieldType) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false;
}

public KeywordFieldType(String name, NamedAnalyzer analyzer) {
Expand All @@ -569,9 +461,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) {
this.scriptValues = null;
this.isDimension = false;
this.isSyntheticSource = false;
this.indexMode = IndexMode.STANDARD;
this.indexSortConfig = null;
this.hasDocValuesSparseIndex = false;
}

@Override
Expand Down Expand Up @@ -962,18 +851,6 @@ public boolean hasScriptValues() {
public boolean hasNormalizer() {
return normalizer != Lucene.KEYWORD_ANALYZER;
}

public IndexMode getIndexMode() {
return indexMode;
}

public IndexSortConfig getIndexSortConfig() {
return indexSortConfig;
}

public boolean hasDocValuesSparseIndex() {
return hasDocValuesSparseIndex;
}
}

private final boolean indexed;
Expand All @@ -989,8 +866,7 @@ public boolean hasDocValuesSparseIndex() {

private final IndexAnalyzers indexAnalyzers;
private final int ignoreAboveDefault;
private final IndexMode indexMode;
private final IndexSortConfig indexSortConfig;
private final int ignoreAbove;

private KeywordFieldMapper(
String simpleName,
Expand All @@ -1014,8 +890,7 @@ private KeywordFieldMapper(
this.indexCreatedVersion = builder.indexCreatedVersion;
this.isSyntheticSource = isSyntheticSource;
this.ignoreAboveDefault = builder.ignoreAboveDefault;
this.indexMode = builder.indexMode;
this.indexSortConfig = builder.indexSortConfig;
this.ignoreAbove = builder.ignoreAbove.getValue();
}

@Override
Expand Down Expand Up @@ -1133,9 +1008,9 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig)
.dimension(fieldType().isDimension())
.init(this);
return new Builder(leafName(), indexAnalyzers, scriptCompiler, ignoreAboveDefault, indexCreatedVersion).dimension(
fieldType().isDimension()
).init(this);
}

@Override
Expand Down
Loading