From f910d77728f2205a8069659d6a5e08b9e299ac52 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 17:30:52 -0400 Subject: [PATCH 1/4] Backport getRandomCompatibleIndexVersion --- ...nticInferenceMetadataFieldsMapperTests.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java index 96a1f90f4e688..2374a7ff9b007 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java @@ -10,9 +10,12 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xpack.inference.InferencePlugin; import java.util.Collection; @@ -42,4 +45,19 @@ public void testFieldHasValueWithEmptyFieldInfos() { public MappedFieldType getMappedFieldType() { return new SemanticInferenceMetadataFieldsMapper.FieldType(); } + + static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) { + if (useLegacyFormat) { + // Randomly choose an index version compatible with the legacy semantic text format + return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.SEMANTIC_TEXT_FIELD_TYPE, IndexVersion.current()); + } else { + // Randomly choose an index version compatible with the new semantic text format + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, + IndexVersion.current() + ); + } + } + } From 319a792bfad9305c635ac7b826f71ac4f29eed6a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 17:34:31 -0400 Subject: [PATCH 2/4] Backport mapper service index version propagation fixes --- .../index/mapper/MapperServiceTestCase.java | 6 +++++ .../mapper/SemanticTextFieldMapperTests.java | 24 ++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 459480d1d7316..0c0016fe2af7d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -207,6 +207,12 @@ protected final MapperService createMapperService(Settings settings, String mapp return mapperService; } + protected final MapperService createMapperService(IndexVersion indexVersion, Settings settings, XContentBuilder mappings) + throws IOException { + MapperService mapperService = createMapperService(indexVersion, settings, () -> true, mappings); + return mapperService; + } + protected final MapperService createMapperService(IndexVersion version, XContentBuilder mapping) throws IOException { return createMapperService(version, getIndexSettings(), () -> true, mapping); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index d78b27607b91b..52d6b5b41bb97 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -24,6 +24,7 @@ import org.apache.lucene.search.join.QueryBitSetProducer; import org.apache.lucene.search.join.ScoreMode; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.CheckedBiFunction; import org.elasticsearch.common.Strings; @@ -33,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; @@ -61,6 +63,7 @@ import org.elasticsearch.search.LeafNestedDocuments; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -112,10 +115,29 @@ protected Collection getPlugins() { } private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); + return createMapperService(mappings, useLegacyFormat, indexVersion, indexVersion); + } + + private MapperService createMapperService( + XContentBuilder mappings, + boolean useLegacyFormat, + IndexVersion minIndexVersion, + IndexVersion maxIndexVersion + ) throws IOException { + validateIndexVersion(minIndexVersion, useLegacyFormat); + IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(random(), minIndexVersion, maxIndexVersion); var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat) .build(); - return createMapperService(settings, mappings); + return createMapperService(indexVersion, settings, mappings); + } + + private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { + if (useLegacyFormat == false && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT)) { + throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format"); + } } @Override From 9e088f7df806bad624b4bb9e3a90c4e6ba6fb9fe Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 17:46:19 -0400 Subject: [PATCH 3/4] Backport testIsEnabled and testIsEnabledByDefault --- ...ticInferenceMetadataFieldsMapperTests.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java index 2374a7ff9b007..7f0af8cbbb3dc 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java @@ -9,9 +9,11 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.index.mapper.InferenceMetadataFieldsMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperServiceTestCase; import org.elasticsearch.plugins.Plugin; @@ -27,6 +29,60 @@ protected Collection getPlugins() { return Collections.singletonList(new InferencePlugin(Settings.EMPTY)); } + public void testIsEnabled() { + var settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .build(); + assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings)); + + // Test that index.mapping.semantic_text.use_legacy_format == false is ignored when the index version is too old to support the new + // format + settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.SEMANTIC_TEXT_FIELD_TYPE, + IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT) + ) // 8.x version range prior to the introduction of the new format + ) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + } + + public void testIsEnabledByDefault() { + // Test upgrades from 8.x + var settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT) + ) + .build(); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); + + settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersion.current()) + ) + .build(); + assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings)); + } + @Override public void testFieldHasValue() { assertTrue( From 993be31f2c013d0868ffc0136b1d82121ddb6718 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 17:50:01 -0400 Subject: [PATCH 4/4] Remove comment --- .../mapper/SemanticInferenceMetadataFieldsMapperTests.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java index 7f0af8cbbb3dc..4c44940c6bf1d 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java @@ -65,7 +65,6 @@ public void testIsEnabled() { } public void testIsEnabledByDefault() { - // Test upgrades from 8.x var settings = Settings.builder() .put( IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(),