From 18eef3f15755913f6284ee5c95e23ef428db71f0 Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 15:39:03 -0400 Subject: [PATCH 1/2] Backport mapper test index version propagation fixes --- .../index/mapper/MapperServiceTestCase.java | 6 ++++ .../mapper/SemanticTextFieldMapperTests.java | 29 +++++++++++++++---- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 459480d1d7316..0c0016fe2af7d 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -207,6 +207,12 @@ protected final MapperService createMapperService(Settings settings, String mapp return mapperService; } + protected final MapperService createMapperService(IndexVersion indexVersion, Settings settings, XContentBuilder mappings) + throws IOException { + MapperService mapperService = createMapperService(indexVersion, settings, () -> true, mappings); + return mapperService; + } + protected final MapperService createMapperService(IndexVersion version, XContentBuilder mapping) throws IOException { return createMapperService(version, getIndexSettings(), () -> true, mapping); } diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java index e837e1b0db989..c973850df8c50 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticTextFieldMapperTests.java @@ -34,6 +34,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.DocumentParsingException; import org.elasticsearch.index.mapper.FieldMapper; @@ -62,6 +63,7 @@ import org.elasticsearch.search.LeafNestedDocuments; import org.elasticsearch.search.NestedDocuments; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.test.index.IndexVersionUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xcontent.json.JsonXContent; @@ -113,14 +115,31 @@ protected Collection getPlugins() { } private MapperService createMapperService(XContentBuilder mappings, boolean useLegacyFormat) throws IOException { + IndexVersion indexVersion = SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat); + return createMapperService(mappings, useLegacyFormat, indexVersion, indexVersion); + } + + private MapperService createMapperService( + XContentBuilder mappings, + boolean useLegacyFormat, + IndexVersion minIndexVersion, + IndexVersion maxIndexVersion + ) throws IOException { + validateIndexVersion(minIndexVersion, useLegacyFormat); + IndexVersion indexVersion = IndexVersionUtils.randomVersionBetween(random(), minIndexVersion, maxIndexVersion); var settings = Settings.builder() - .put( - IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), - SemanticInferenceMetadataFieldsMapperTests.getRandomCompatibleIndexVersion(useLegacyFormat) - ) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), indexVersion) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), useLegacyFormat) .build(); - return createMapperService(settings, mappings); + return createMapperService(indexVersion, settings, mappings); + } + + private static void validateIndexVersion(IndexVersion indexVersion, boolean useLegacyFormat) { + if (useLegacyFormat == false + && indexVersion.before(IndexVersions.INFERENCE_METADATA_FIELDS) + && indexVersion.between(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0) == false) { + throw new IllegalArgumentException("Index version " + indexVersion + " does not support new semantic text format"); + } } @Override From f84716af575cb4b6c88692785fa4e263a829228a Mon Sep 17 00:00:00 2001 From: Mike Pellegrini Date: Thu, 8 May 2025 15:49:32 -0400 Subject: [PATCH 2/2] Backport getRandomCompatibleIndexVersion changes --- ...ticInferenceMetadataFieldsMapperTests.java | 52 +++++++++++++++---- 1 file changed, 41 insertions(+), 11 deletions(-) diff --git a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java index 23519ec86cbc4..ca19ef0171825 100644 --- a/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java +++ b/x-pack/plugin/inference/src/test/java/org/elasticsearch/xpack/inference/mapper/SemanticInferenceMetadataFieldsMapperTests.java @@ -37,22 +37,44 @@ public void testIsEnabled() { assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); settings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(true)) - .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) .build(); assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); settings = Settings.builder() .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) - .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), true) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) + .build(); + assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings)); + + // Test that index.mapping.semantic_text.use_legacy_format == false is ignored when the index version is too old to support the new + // format + settings = Settings.builder() + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.SEMANTIC_TEXT_FIELD_TYPE, + IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT) + ) // 8.x version range prior to the introduction of the new format + ) + .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) .build(); assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); settings = Settings.builder() - .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), getRandomCompatibleIndexVersion(false)) + .put( + IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), + IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.UPGRADE_TO_LUCENE_10_0_0, + IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS) + ) // 9.x version range prior to the introduction of the new format + ) .put(InferenceMetadataFieldsMapper.USE_LEGACY_SEMANTIC_TEXT_FORMAT.getKey(), false) .build(); - assertTrue(InferenceMetadataFieldsMapper.isEnabled(settings)); + assertFalse(InferenceMetadataFieldsMapper.isEnabled(settings)); } public void testIsEnabledByDefault() { @@ -115,18 +137,26 @@ public MappedFieldType getMappedFieldType() { static IndexVersion getRandomCompatibleIndexVersion(boolean useLegacyFormat) { if (useLegacyFormat) { + // Randomly choose an index version compatible with the legacy semantic text format if (randomBoolean()) { - return IndexVersionUtils.randomVersionBetween( - random(), - IndexVersions.UPGRADE_TO_LUCENE_10_0_0, - IndexVersionUtils.getPreviousVersion(IndexVersions.INFERENCE_METADATA_FIELDS) - ); + // 9.x+ version + return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.UPGRADE_TO_LUCENE_10_0_0, IndexVersion.current()); } - return IndexVersionUtils.randomPreviousCompatibleVersion(random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT); + + // 8.x version + return IndexVersionUtils.randomVersionBetween( + random(), + IndexVersions.SEMANTIC_TEXT_FIELD_TYPE, + IndexVersionUtils.getPreviousVersion(IndexVersions.UPGRADE_TO_LUCENE_10_0_0) + ); } else { + // Randomly choose an index version compatible with the new semantic text format if (randomBoolean()) { + // 9.x+ version return IndexVersionUtils.randomVersionBetween(random(), IndexVersions.INFERENCE_METADATA_FIELDS, IndexVersion.current()); } + + // 8.x version return IndexVersionUtils.randomVersionBetween( random(), IndexVersions.INFERENCE_METADATA_FIELDS_BACKPORT,