diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index a5045e88a3bdb..cb9bc55485c91 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -104,8 +104,15 @@ public static class Builder extends FieldMapper.Builder { private final TextParams.Analyzers analyzers; private final boolean withinMultiField; + private final boolean storedFieldInBinaryFormat; - public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers, boolean withinMultiField) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + IndexAnalyzers indexAnalyzers, + boolean withinMultiField, + boolean storedFieldInBinaryFormat + ) { super(name); this.indexCreatedVersion = indexCreatedVersion; this.analyzers = new TextParams.Analyzers( @@ -115,6 +122,7 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind indexCreatedVersion ); this.withinMultiField = withinMultiField; + this.storedFieldInBinaryFormat = storedFieldInBinaryFormat; } @Override @@ -134,7 +142,8 @@ private MatchOnlyTextFieldType buildFieldType(MapperBuilderContext context) { context.isSourceSynthetic(), meta.getValue(), withinMultiField, - multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() + multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField(), + storedFieldInBinaryFormat ); return ft; } @@ -154,8 +163,18 @@ public MatchOnlyTextFieldMapper build(MapperBuilderContext context) { } } + private static boolean isSyntheticSourceStoredFieldInBinaryFormat(IndexVersion indexCreatedVersion) { + return indexCreatedVersion.onOrAfter(IndexVersions.MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X); + } + public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers(), c.isWithinMultiField()) + (n, c) -> new Builder( + n, + c.indexVersionCreated(), + c.getIndexAnalyzers(), + c.isWithinMultiField(), + isSyntheticSourceStoredFieldInBinaryFormat(c.indexVersionCreated()) + ) ); public static class MatchOnlyTextFieldType extends StringFieldType { @@ -166,6 +185,7 @@ public static class MatchOnlyTextFieldType extends StringFieldType { private final boolean withinMultiField; private final boolean hasCompatibleMultiFields; + private final boolean storedFieldInBinaryFormat; public MatchOnlyTextFieldType( String name, @@ -174,7 +194,8 @@ public MatchOnlyTextFieldType( boolean isSyntheticSource, Map meta, boolean withinMultiField, - boolean hasCompatibleMultiFields + boolean hasCompatibleMultiFields, + boolean storedFieldInBinaryFormat ) { super(name, true, false, false, tsi, meta); this.indexAnalyzer = Objects.requireNonNull(indexAnalyzer); @@ -182,6 +203,7 @@ public MatchOnlyTextFieldType( this.originalName = isSyntheticSource ? name() + "._original" : null; this.withinMultiField = withinMultiField; this.hasCompatibleMultiFields = hasCompatibleMultiFields; + this.storedFieldInBinaryFormat = storedFieldInBinaryFormat; } public MatchOnlyTextFieldType(String name) { @@ -192,6 +214,7 @@ public MatchOnlyTextFieldType(String name) { false, Collections.emptyMap(), false, + false, false ); } @@ -450,7 +473,11 @@ protected BytesRef toBytesRef(Object v) { @Override public BlockLoader blockLoader(BlockLoaderContext blContext) { if (textFieldType.isSyntheticSource()) { - return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource()); + if (storedFieldInBinaryFormat) { + return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource()); + } else { + return new BytesFromMixedStringsBytesRefBlockLoader(storedFieldNameForSyntheticSource()); + } } SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name())); // MatchOnlyText never has norms, so we have to use the field names field @@ -501,6 +528,7 @@ private String storedFieldNameForSyntheticSource() { private final boolean storeSource; private final FieldType fieldType; private final boolean withinMultiField; + private final boolean storedFieldInBinaryFormat; private MatchOnlyTextFieldMapper( String simpleName, @@ -520,6 +548,7 @@ private MatchOnlyTextFieldMapper( this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue(); this.storeSource = storeSource; this.withinMultiField = builder.withinMultiField; + this.storedFieldInBinaryFormat = builder.storedFieldInBinaryFormat; } @Override @@ -529,7 +558,7 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField).init(this); + return new Builder(leafName(), indexCreatedVersion, indexAnalyzers, withinMultiField, storedFieldInBinaryFormat).init(this); } @Override @@ -546,8 +575,12 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio context.addToFieldNames(fieldType().name()); if (storeSource) { - final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); - context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef)); + if (storedFieldInBinaryFormat) { + final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length()); + context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef)); + } else { + context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value.string())); + } } } diff --git a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java index 39b41f121299e..37521966c4a1f 100644 --- a/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java +++ b/modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java @@ -26,8 +26,10 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.LuceneDocument; @@ -356,10 +358,14 @@ public void testStoreParameterDefaultsSyntheticSourceTextFieldIsMultiField() thr } public void testLoadSyntheticSourceFromStringOrBytesRef() throws IOException { - DocumentMapper mapper = createSytheticSourceMapperService(mapping(b -> { + var mappings = mapping(b -> { b.startObject("field1").field("type", "match_only_text").endObject(); b.startObject("field2").field("type", "match_only_text").endObject(); - })).documentMapper(); + }); + var settings = Settings.builder().put("index.mapping.source.mode", "synthetic").build(); + DocumentMapper mapper = createMapperService(IndexVersions.UPGRADE_TO_LUCENE_9_12_2, settings, () -> true, mappings) + .documentMapper(); + try (Directory directory = newDirectory()) { RandomIndexWriter iw = indexWriterForSyntheticSource(directory); diff --git a/server/src/main/java/org/elasticsearch/index/IndexVersions.java b/server/src/main/java/org/elasticsearch/index/IndexVersions.java index 1a592163b9b53..56273f6946655 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexVersions.java +++ b/server/src/main/java/org/elasticsearch/index/IndexVersions.java @@ -136,6 +136,7 @@ private static IndexVersion def(int id, Version luceneVersion) { public static final IndexVersion MAPPER_TEXT_MATCH_ONLY_MULTI_FIELDS_DEFAULT_NOT_STORED_8_19 = def(8_533_0_00, Version.LUCENE_9_12_1); public static final IndexVersion UPGRADE_TO_LUCENE_9_12_2 = def(8_534_0_00, Version.LUCENE_9_12_2); public static final IndexVersion SPARSE_VECTOR_PRUNING_INDEX_OPTIONS_SUPPORT_BACKPORT_8_X = def(8_535_0_00, Version.LUCENE_9_12_2); + public static final IndexVersion MATCH_ONLY_TEXT_STORED_AS_BYTES_BACKPORT_8_X = def(8_536_0_00, Version.LUCENE_9_12_2); /* * STOP! READ THIS FIRST! No, really,