Skip to content

[8.19] Add index version for match_only_text stored field in binary format (#130363) #130416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jul 2, 2025
Merged

[8.19] Add index version for match_only_text stored field in binary format (#130363) #130416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4fea54f
Add index version for match_only_text stored field in binary format (…
jordan-powers Jul 2, 2025
b3297e1
Fix IndexVersion in testLoadSyntheticSourceFromStringOrBytesRef
jordan-powers Jul 2, 2025
f7a7a00
Trigger build for auto-merge
jordan-powers Jul 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 41 additions & 8 deletions ...-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,15 @@ public static class Builder extends FieldMapper.Builder {

private final TextParams.Analyzers analyzers;
private final boolean withinMultiField;
private final boolean storedFieldInBinaryFormat;

public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) {
public Builder(
String name,
IndexVersion indexCreatedVersion,
IndexAnalyzers indexAnalyzers,
boolean withinMultiField,
boolean storedFieldInBinaryFormat
) {
super(name);
this.indexCreatedVersion = indexCreatedVersion;
this.analyzers = new TextParams.Analyzers(
Expand All @@ -115,6 +122,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind
indexCreatedVersion
);
this.withinMultiField = withinMultiField;
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
}

@Override
Expand All @@ -134,7 +142,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) {
context.isSourceSynthetic(),
meta.getValue(),
withinMultiField,
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField()
multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(),
storedFieldInBinaryFormat
);
return ft;
}
Expand All @@ -154,8 +163,18 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) {
}
}

private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion indexCreatedVersion) {
return indexCreatedVersion.onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X);
}

public static final TypeParser PARSER = new TypeParser(
(n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField())
(n, c) -> new Builder(
n,
c.indexVersionCreated(),
c.getIndexAnalyzers(),
c.isWithinMultiField(),
isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated())
)
);

public static class MatchOnlyTextFieldType extends StringFieldType {
Expand All @@ -166,6 +185,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType {

private final boolean withinMultiField;
private final boolean hasCompatibleMultiFields;
private final boolean storedFieldInBinaryFormat;

public MatchOnlyTextFieldType(
String name,
Expand All @@ -174,14 +194,16 @@ public MatchOnlyTextFieldType(
boolean isSyntheticSource,
Map<String, String> meta,
boolean withinMultiField,
boolean hasCompatibleMultiFields
boolean hasCompatibleMultiFields,
boolean storedFieldInBinaryFormat
) {
super(name, true, false, false, tsi, meta);
this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer);
this.textFieldType = new TextFieldType(name, isSyntheticSource);
this.originalName = isSyntheticSource ? name() + "._original" : null;
this.withinMultiField = withinMultiField;
this.hasCompatibleMultiFields = hasCompatibleMultiFields;
this.storedFieldInBinaryFormat = storedFieldInBinaryFormat;
}

public MatchOnlyTextFieldType(String name) {
Expand All @@ -192,6 +214,7 @@ public MatchOnlyTextFieldType(String name) {
false,
Collections.emptyMap(),
false,
false,
false
);
}
Expand Down Expand Up @@ -450,7 +473,11 @@ protected BytesRef toBytesRef(Object v) {
@Override
public BlockLoader blockLoader(BlockLoaderContext blContext) {
if (textFieldType.isSyntheticSource()) {
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
if (storedFieldInBinaryFormat) {
return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
} else {
return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource());
}
}
SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
// MatchOnlyText never has norms, so we have to use the field names field
Expand Down Expand Up @@ -501,6 +528,7 @@ private String storedFieldNameForSyntheticSource() {
private final boolean storeSource;
private final FieldType fieldType;
private final boolean withinMultiField;
private final boolean storedFieldInBinaryFormat;

private MatchOnlyTextFieldMapper(
String simpleName,
Expand All @@ -520,6 +548,7 @@ private MatchOnlyTextFieldMapper(
this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue();
this.storeSource = storeSource;
this.withinMultiField = builder.withinMultiField;
this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat;
}

@Override
Expand All @@ -529,7 +558,7 @@ public Map<String, NamedAnalyzer> indexAnalyzers() {

@Override
public FieldMapper.Builder getMergeBuilder() {
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this);
return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this);
}

@Override
Expand All @@ -546,8 +575,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio
context.addToFieldNames(fieldType().name());

if (storeSource) {
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
if (storedFieldInBinaryFormat) {
final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
} else {
context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string()));
}
}
}

Expand Down
10 changes: 8 additions & 2 deletions ...as/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,10 @@
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.Tuple;
import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.index.IndexVersions;
import org.elasticsearch.index.mapper.DocumentMapper;
import org.elasticsearch.index.mapper.KeywordFieldMapper;
import org.elasticsearch.index.mapper.LuceneDocument;
Expand Down Expand Up @@ -356,10 +358,14 @@ public void testStoreParameterDefaultsSyntheticSourceTextFieldIsMultiField() thr
}

public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException {
DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> {
var mappings = mapping(b -> {
b.startObject("field1").field("type", "match_only_text").endObject();
b.startObject("field2").field("type", "match_only_text").endObject();
})).documentMapper();
});
var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build();
DocumentMapper mapper = createMapperService(IndexVersions.UPGRADE_TO_LUCENE_9_12_2, settings, () -> true, mappings)
.documentMapper();

try (Directory directory = newDirectory()) {
RandomIndexWriter iw = indexWriterForSyntheticSource(directory);

Expand Down
1 change: 1 addition & 0 deletions server/src/main/java/org/elasticsearch/index/IndexVersions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ private static IndexVersion def(int id, Version luceneVersion) {
public static final IndexVersion MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19 = def(8_533_0_00, Version.LUCENE_9_12_1);
public static final IndexVersion UPGRADE_TO_LUCENE_9_12_2 = def(8_534_0_00, Version.LUCENE_9_12_2);
public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X = def(8_535_0_00, Version.LUCENE_9_12_2);
public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X = def(8_536_0_00, Version.LUCENE_9_12_2);

/*
* STOP! READ THIS FIRST! No, really,
Expand Down