From ad9200452da349a1427e22ae47081afef469a656 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 10 Nov 2025 15:07:04 -0800 Subject: [PATCH 1/6] Update upgrade assistant API to account for percolator fields (#137548) This change modifies the deprecation API to check for any indices prior to 9.latest that have percolator fields in the mappings. Any that do are then returned as required to be reindexed or deleted. Read-only is excluded as this would not fix the underlying transport version issue. Note this change only concerns user created indices. System indices will need to be addressed separately as part of the migration API, but should be able to share code. ES-13234 --- .../cluster/metadata/IndexMetadata.java | 10 + .../metadata/MetadataCreateIndexService.java | 3 + .../common/settings/IndexScopedSettings.java | 1 + .../deprecation/DeprecatedIndexPredicate.java | 133 ++++++++++- .../DataStreamDeprecationChecker.java | 50 +++- .../deprecation/IndexDeprecationChecker.java | 216 ++++++++---------- .../AbstractWatcherIntegrationTestCase.java | 1 + 7 files changed, 289 insertions(+), 125 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 094626018d449..59001d7f28202 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -376,6 +376,16 @@ public static APIBlock readFrom(StreamInput input) throws IOException { Property.PrivateIndex ); + public static final String SETTING_TRANSPORT_VERSION_CREATED = "index.transport_version.created"; + + public static final Setting SETTING_INDEX_TRANSPORT_VERSION_CREATED = Setting.versionIdSetting( + SETTING_TRANSPORT_VERSION_CREATED, + TransportVersion.fromId(0), + TransportVersion::fromId, + Property.IndexScope, + Property.PrivateIndex + ); + public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string"; public static final String SETTING_CREATION_DATE = "index.creation_date"; diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 44bf6b93c014f..2fd4b7971c774 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -1223,6 +1223,9 @@ static Settings aggregateIndexSettings( } indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); + if (indexSettingsBuilder.get(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED) == null) { + indexSettingsBuilder.put(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED, TransportVersion.current()); + } if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) { indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 023c532798082..ef9f436cc0fe7 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -65,6 +65,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, IndexMetadata.SETTING_INDEX_VERSION_CREATED, + IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED, IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY, IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING, IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index 78a9029165817..6f5b37a7cf375 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -14,6 +15,11 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; public class DeprecatedIndexPredicate { @@ -53,7 +59,7 @@ public static Predicate getReindexRequiredPredicate( * if false, only those without a block are returned * @param includeSystem if true, all indices including system will be returned, * if false, only non-system indices are returned - * @return a predicate that returns true for indices that need to be reindexed + * @return returns true for indices that need to be reindexed */ public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) @@ -62,6 +68,59 @@ && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + /** + * This method checks if this index is on the current transport version for the current minor release version. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns true for indices that need to be reindexed + */ + public static boolean reindexRequiredForTransportVersion( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + return transportVersionBeforeCurrentMinorRelease(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) + && isNotSearchableSnapshot(indexMetadata) + && matchBlockedStatus(indexMetadata, filterToBlockedStatus); + } + + /** + * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version + * for the current minor release. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns a message as a string for each incompatible percolator field found + */ + public static List reindexRequiredForPecolatorFields( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + List percolatorIncompatibleFieldMappings = new ArrayList<>(); + if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) { + percolatorIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + indexMetadata.mapping().type(), + indexMetadata.mapping().sourceAsMap(), + property -> "percolator".equals(property.get("type")), + (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]", + "", + "" + ) + ); + } + return percolatorIncompatibleFieldMappings; + } + private static boolean isNotSystem(IndexMetadata indexMetadata) { return indexMetadata.isSystem() == false; } @@ -77,4 +136,76 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus; } + + private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { + // We divide each transport version by 1000 to get the base id. + return IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED.get(indexMetadata.getSettings()).id() / 1000 < TransportVersion + .current() + .id() / 1000; + } + + /** + * iterates through the "properties" field of mappings and returns any predicates that match in the + * form of issue-strings. + * + * @param type the document type + * @param parentMap the mapping to read properties from + * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true + * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation + * @return a list of issues found in fields + */ + @SuppressWarnings("unchecked") + public static List findInPropertiesRecursively( + String type, + Map parentMap, + Function, Boolean> predicate, + BiFunction, String> fieldFormatter, + String fieldBeginMarker, + String fieldEndMarker + ) { + List issues = new ArrayList<>(); + Map properties = (Map) parentMap.get("properties"); + if (properties == null) { + return issues; + } + for (Map.Entry entry : properties.entrySet()) { + Map valueMap = (Map) entry.getValue(); + if (predicate.apply(valueMap)) { + issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); + } + + Map values = (Map) valueMap.get("fields"); + if (values != null) { + for (Map.Entry multifieldEntry : values.entrySet()) { + Map multifieldValueMap = (Map) multifieldEntry.getValue(); + if (predicate.apply(multifieldValueMap)) { + issues.add( + fieldBeginMarker + + fieldFormatter.apply(type, entry) + + ", multifield: " + + multifieldEntry.getKey() + + fieldEndMarker + ); + } + if (multifieldValueMap.containsKey("properties")) { + issues.addAll( + findInPropertiesRecursively( + type, + multifieldValueMap, + predicate, + fieldFormatter, + fieldBeginMarker, + fieldEndMarker + ) + ); + } + } + } + if (valueMap.containsKey("properties")) { + issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); + } + } + + return issues; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index b05bdb25df792..9b8f52f61617f 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -88,9 +88,24 @@ public Map> check(ProjectMetadata project) { static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); - + Set percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, false); + if (percolatorIndicesNeedingUpgrade.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()), + entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade) + ) + ); + } Set indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, project, false); - if (indicesNeedingUpgrade.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -112,6 +127,23 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata p static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); + Set percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, true); + if (percolatorIgnoredIndices.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", + "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()), + entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices) + ) + ); + } Set ignoredIndices = getReindexRequiredIndices(backingIndices, project, true); if (ignoredIndices.isEmpty() == false) { return new DeprecationIssue( @@ -144,6 +176,20 @@ private static Set getReindexRequiredIndices( .collect(Collectors.toUnmodifiableSet()); } + private static Set getReindexRequiredIndicesWithPercolatorFields( + List backingIndices, + ProjectMetadata project, + boolean filterToBlockedStatus + ) { + return backingIndices.stream() + .filter( + index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields(project.index(index), filterToBlockedStatus, false) + .isEmpty() == false + ) + .map(Index::getName) + .collect(Collectors.toUnmodifiableSet()); + } + @Override public String getName() { return NAME; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 5f1a3310027c3..8a841afc46371 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -29,8 +29,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; @@ -93,37 +91,56 @@ private DeprecationIssue oldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - // TODO: this check needs to be revised. It's trivially true right now. - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and requires action before upgrading to %d.0. The following transforms are " - + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " - + "transforms destination indices for your upgrade.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + false, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://ela.st/es-deprecation-9-index-version", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to %d.0. The following " + + "transforms are configured to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to prepare transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://ela.st/es-deprecation-9-index-version", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -137,40 +154,60 @@ private DeprecationIssue ignoredOldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " - + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " - + "about how to handle your transforms destination indices.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + true, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", - "https://ela.st/es-deprecation-9-index-version", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in " - + Version.CURRENT.major - + ".0", + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", + "https://ela.st/es-deprecation-9-index-version", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in " + + Version.CURRENT.major + + ".0", + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -272,71 +309,6 @@ private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer findInPropertiesRecursively( - String type, - Map parentMap, - Function, Boolean> predicate, - BiFunction, String> fieldFormatter, - String fieldBeginMarker, - String fieldEndMarker - ) { - List issues = new ArrayList<>(); - Map properties = (Map) parentMap.get("properties"); - if (properties == null) { - return issues; - } - for (Map.Entry entry : properties.entrySet()) { - Map valueMap = (Map) entry.getValue(); - if (predicate.apply(valueMap)) { - issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); - } - - Map values = (Map) valueMap.get("fields"); - if (values != null) { - for (Map.Entry multifieldEntry : values.entrySet()) { - Map multifieldValueMap = (Map) multifieldEntry.getValue(); - if (predicate.apply(multifieldValueMap)) { - issues.add( - fieldBeginMarker - + fieldFormatter.apply(type, entry) - + ", multifield: " - + multifieldEntry.getKey() - + fieldEndMarker - ); - } - if (multifieldValueMap.containsKey("properties")) { - issues.addAll( - findInPropertiesRecursively( - type, - multifieldValueMap, - predicate, - fieldFormatter, - fieldBeginMarker, - fieldEndMarker - ) - ); - } - } - } - if (valueMap.containsKey("properties")) { - issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); - } - } - - return issues; - } - private DeprecationIssue deprecatedCamelCasePattern( IndexMetadata indexMetadata, ProjectMetadata project, @@ -346,7 +318,7 @@ private DeprecationIssue deprecatedCamelCasePattern( fieldLevelMappingIssue( indexMetadata, ((mappingMetadata, sourceAsMap) -> fields.addAll( - findInPropertiesRecursively( + DeprecatedIndexPredicate.findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, this::isDateFieldWithCamelCasePattern, diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index d237e08b30c19..47215174d0639 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -275,6 +275,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia newSettings.remove("index.uuid"); newSettings.remove("index.creation_date"); newSettings.remove("index.version.created"); + newSettings.remove("index.transport_version.created"); assertAcked(indicesAdmin().prepareCreate(to).setMapping(mapping.sourceAsMap()).setSettings(newSettings)); ensureGreen(to); From 36449df36eac5e0dd5172143302e5fdbf01c81e8 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Thu, 13 Nov 2025 15:38:10 -0800 Subject: [PATCH 2/6] replace setting with direct field --- .../cluster/metadata/IndexMetadata.java | 53 +++++++++++++++---- .../metadata/MetadataCreateIndexService.java | 4 +- .../common/settings/IndexScopedSettings.java | 1 - .../index_created_transport_version.csv | 1 + .../resources/transport/upper_bounds/9.3.csv | 2 +- .../cluster/metadata/IndexMetadataTests.java | 5 ++ .../deprecation/DeprecatedIndexPredicate.java | 4 +- 7 files changed, 52 insertions(+), 18 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 59001d7f28202..f25031f1ce3d8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -153,6 +153,7 @@ public class IndexMetadata implements Diffable, ToXContentFragmen public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; private static final TransportVersion INDEX_RESHARDING_METADATA = TransportVersion.fromName("index_resharding_metadata"); + private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version"); @Nullable public String getDownsamplingInterval() { @@ -376,16 +377,6 @@ public static APIBlock readFrom(StreamInput input) throws IOException { Property.PrivateIndex ); - public static final String SETTING_TRANSPORT_VERSION_CREATED = "index.transport_version.created"; - - public static final Setting SETTING_INDEX_TRANSPORT_VERSION_CREATED = Setting.versionIdSetting( - SETTING_TRANSPORT_VERSION_CREATED, - TransportVersion.fromId(0), - TransportVersion::fromId, - Property.IndexScope, - Property.PrivateIndex - ); - public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string"; public static final String SETTING_CREATION_DATE = "index.creation_date"; @@ -593,6 +584,7 @@ public Iterator> settings() { public static final List PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN); static final String KEY_VERSION = "version"; + static final String KEY_TRANSPORT_VERSION = "transport_version"; static final String KEY_MAPPING_VERSION = "mapping_version"; static final String KEY_SETTINGS_VERSION = "settings_version"; static final String KEY_ALIASES_VERSION = "aliases_version"; @@ -633,6 +625,7 @@ public Iterator> settings() { private final Index index; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; @@ -720,6 +713,7 @@ public Iterator> settings() { private IndexMetadata( final Index index, final long version, + final TransportVersion transportVersion, final long mappingVersion, final long settingsVersion, final long aliasesVersion, @@ -771,6 +765,7 @@ private IndexMetadata( ) { this.index = index; this.version = version; + this.transportVersion = transportVersion; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; this.mappingsUpdatedVersion = mappingsUpdatedVersion; @@ -838,6 +833,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -902,6 +898,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -974,6 +971,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1037,6 +1035,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1095,6 +1094,7 @@ public IndexMetadata withIncrementedVersion() { return new IndexMetadata( this.index, this.version + 1, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1158,6 +1158,10 @@ public long getVersion() { return this.version; } + public TransportVersion getTransportVersion() { + return transportVersion; + } + public long getMappingVersion() { return mappingVersion; } @@ -1639,6 +1643,7 @@ private static class IndexMetadataDiff implements Diff { private final String index; private final int routingNumShards; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; private final long settingsVersion; private final long aliasesVersion; @@ -1672,6 +1677,7 @@ private static class IndexMetadataDiff implements Diff { IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); version = after.version; + transportVersion = after.transportVersion; mappingVersion = after.mappingVersion; settingsVersion = after.settingsVersion; aliasesVersion = after.aliasesVersion; @@ -1725,6 +1731,9 @@ private static class IndexMetadataDiff implements Diff { index = in.readString(); routingNumShards = in.readInt(); version = in.readLong(); + transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0); mappingVersion = in.readVLong(); settingsVersion = in.readVLong(); aliasesVersion = in.readVLong(); @@ -1792,6 +1801,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeInt(routingNumShards); out.writeLong(version); + if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) { + TransportVersion.writeVersion(transportVersion, out); + } out.writeVLong(mappingVersion); out.writeVLong(settingsVersion); out.writeVLong(aliasesVersion); @@ -1832,6 +1844,7 @@ public void writeTo(StreamOutput out) throws IOException { public IndexMetadata apply(IndexMetadata part) { Builder builder = builder(index); builder.version(version); + builder.transportVersion(transportVersion); builder.mappingVersion(mappingVersion); builder.settingsVersion(settingsVersion); builder.aliasesVersion(aliasesVersion); @@ -1875,6 +1888,9 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { public static IndexMetadata readFrom(StreamInput in, @Nullable Function mappingLookup) throws IOException { Builder builder = new Builder(in.readString()); builder.version(in.readLong()); + builder.transportVersion( + in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) ? TransportVersion.readVersion(in) : TransportVersion.fromId(0) + ); builder.mappingVersion(in.readVLong()); builder.settingsVersion(in.readVLong()); builder.aliasesVersion(in.readVLong()); @@ -1942,6 +1958,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.state(State.fromString(parser.text())); case KEY_VERSION -> builder.version(parser.longValue()); + case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text())); case KEY_MAPPING_VERSION -> { mappingVersion = true; builder.mappingVersion(parser.longValue()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 2fd4b7971c774..952b3080d3713 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -627,6 +627,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.transportVersion(TransportVersion.current()); tmpImdBuilder.system(isSystem); // Set up everything, now locally create the index to see that things are ok, and apply @@ -1223,9 +1224,6 @@ static Settings aggregateIndexSettings( } indexSettingsBuilder.put(IndexMetadata.SETTING_VERSION_CREATED, createdVersion); - if (indexSettingsBuilder.get(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED) == null) { - indexSettingsBuilder.put(IndexMetadata.SETTING_TRANSPORT_VERSION_CREATED, TransportVersion.current()); - } if (INDEX_NUMBER_OF_SHARDS_SETTING.exists(indexSettingsBuilder) == false) { indexSettingsBuilder.put(SETTING_NUMBER_OF_SHARDS, INDEX_NUMBER_OF_SHARDS_SETTING.get(settings)); } diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index ef9f436cc0fe7..023c532798082 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -65,7 +65,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING, MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, IndexMetadata.SETTING_INDEX_VERSION_CREATED, - IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED, IndexMetadata.SETTING_INDEX_VERSION_COMPATIBILITY, IndexMetadata.INDEX_ROUTING_EXCLUDE_GROUP_SETTING, IndexMetadata.INDEX_ROUTING_INCLUDE_GROUP_SETTING, diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv new file mode 100644 index 0000000000000..fad15855ad264 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -0,0 +1 @@ +9217000 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 28e1d47db658d..66646eeee701a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -esql_execution_metadata,9216000 +index_created_transport_version,9217000 diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d0aae463dd193..3f4410a29a137 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -151,6 +152,7 @@ public void testIndexMetadataSerialization() throws IOException { ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); @@ -178,6 +180,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata, deserialized); assertEquals(metadata.hashCode(), deserialized.hashCode()); + assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion()); @@ -214,6 +217,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -282,6 +286,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro fromXContentMeta ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index 6f5b37a7cf375..bc63a10f0fbd1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -139,9 +139,7 @@ private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean f private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { // We divide each transport version by 1000 to get the base id. - return IndexMetadata.SETTING_INDEX_TRANSPORT_VERSION_CREATED.get(indexMetadata.getSettings()).id() / 1000 < TransportVersion - .current() - .id() / 1000; + return indexMetadata.getTransportVersion().id() / 1000 < TransportVersion.current().id() / 1000; } /** From 2b109d9e2d2d9dfcff38b8a73d5dcc479fb70267 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 14 Nov 2025 09:48:48 -0800 Subject: [PATCH 3/6] Add unit tests --- .../DataStreamDeprecationChecker.java | 10 +- .../DataStreamDeprecationCheckerTests.java | 169 ++++++++++++++++-- .../IndexDeprecationCheckerTests.java | 120 +++++++++++++ 3 files changed, 282 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index 9b8f52f61617f..4925d4004e135 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -93,8 +93,9 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata p return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", - "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", - "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", false, ofEntries( entry("reindex_required", true), @@ -132,8 +133,9 @@ static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ProjectMet return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", - "https://www.elastic.co/guide/en/elasticsearch/reference/8.19/percolator.html#_reindexing_your_percolator_queries", - "The data stream was created before 8.19 and contains mappings that must be reindexed due to containing percolator fields.", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", false, ofEntries( entry("reindex_required", true), diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 49611209862a3..87975f53b77dd 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -7,16 +7,19 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; @@ -73,6 +76,45 @@ public void testOldIndicesCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesCheckWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); + DataStream dataStream = createTestDataStream(oldIndexCount, 0, newIndexCount, 0, nameToIndexMetadata, expectedIndices, + mappingMetadata, randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("indices_requiring_upgrade_count", expectedIndices.size()), + entry("indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.size(), equalTo(1)); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldIndicesCheckWithOnlyNewIndices() { // This tests what happens when any old indices that we have are closed. We expect no deprecation warning. int newOpenIndexCount = randomIntBetween(1, 100); @@ -105,6 +147,8 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { Map nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); + MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); DataStream dataStream = createTestDataStream( oldOpenIndexCount, oldClosedIndexCount, @@ -138,6 +182,18 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + private DataStream createTestDataStream( + int oldOpenIndexCount, + int oldClosedIndexCount, + int newOpenIndexCount, + int newClosedIndexCount, + Map nameToIndexMetadata, + Set expectedIndices + ) { + return createTestDataStream(oldOpenIndexCount, oldClosedIndexCount, newOpenIndexCount, newClosedIndexCount, nameToIndexMetadata, + expectedIndices, MappingMetadata.EMPTY_MAPPINGS, TransportVersion.fromId(0)); + } + /* * This creates a test DataStream with the given counts. The nameToIndexMetadata Map and the expectedIndices Set are mutable collections * that will be populated by this method. @@ -148,21 +204,23 @@ private DataStream createTestDataStream( int newOpenIndexCount, int newClosedIndexCount, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion oldTransportVersion ) { List allIndices = new ArrayList<>(); for (int i = 0; i < oldOpenIndexCount; i++) { - allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < oldClosedIndexCount; i++) { - allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < newOpenIndexCount; i++) { - allIndices.add(createNewIndex(i, false, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, false, nameToIndexMetadata, mappingMetadata)); } for (int i = 0; i < newClosedIndexCount; i++) { - allIndices.add(createNewIndex(i, true, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, true, nameToIndexMetadata, mappingMetadata)); } DataStream dataStream = new DataStream( @@ -188,13 +246,16 @@ private Index createOldIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { - return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices); + return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata) { - return createIndex(false, suffix, isClosed, nameToIndexMetadata, null); + private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata, + MappingMetadata mappingMetadata) { + return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } private Index createIndex( @@ -202,7 +263,9 @@ private Index createIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { Settings.Builder settingsBuilder = isOld ? settings(IndexVersion.fromId(7170099)) : settings(IndexVersion.current()); String indexName = (isOld ? "old-" : "new-") + (isClosed ? "closed-" : "") + "data-stream-index-" + suffix; @@ -216,7 +279,9 @@ private Index createIndex( IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(settingsBuilder) .numberOfShards(1) - .numberOfReplicas(0); + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(transportVersion); if (isClosed) { indexMetadataBuilder.state(IndexMetadata.State.CLOSE); } @@ -251,7 +316,7 @@ public void testOldIndicesIgnoredWarningCheck() { } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } @@ -298,6 +363,84 @@ public void testOldIndicesIgnoredWarningCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesIgnoredWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); + + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(9000019)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + expectedIndices.add(indexName); + + Settings.Builder settingsBuilder = settings; + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + + for (int i = 0; i < newIndexCount; i++) { + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, mappingMetadata); + allIndices.add(newIndex); + } + + DataStream dataStream = new DataStream( + randomAlphaOfLength(10), + allIndices, + randomNegativeLong(), + Map.of(), + randomBoolean(), + false, + false, + randomBoolean(), + randomFrom(IndexMode.values()), + null, + randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null), + List.of(), + randomBoolean(), + null + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()), + entry("ignored_indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldSystemDataStreamIgnored() { // We do not want system data streams coming back in the deprecation info API int oldIndexCount = randomIntBetween(1, 100); @@ -319,7 +462,7 @@ public void testOldSystemDataStreamIgnored() { nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } DataStream dataStream = new DataStream( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 5ea8a9eea6b8e..9beb45566abc8 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -10,12 +10,14 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamMetadata; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -42,6 +44,8 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.nullValue; +import static org.mockito.ArgumentMatchers.isNull; public class IndexDeprecationCheckerTests extends ESTestCase { @@ -191,6 +195,80 @@ public void testMultipleOldIndicesCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping( + """ + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """ + ) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + + public void testLatestIndicesCheckWithPercolatorFields() { + IndexVersion latestVersion = IndexVersion.current(); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(latestVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping( + """ + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """ + ) + .transportVersion(TransportVersion.current()) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex.isEmpty(), equalTo(true)); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -405,6 +483,48 @@ public void testMultipleOldIndicesIgnoredCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIgnoreIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping( + """ + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """ + ) + .transportVersion(TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); From 57a2a8b1e6293fec75676b6dd15a96d5df27acb4 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 14 Nov 2025 10:10:00 -0800 Subject: [PATCH 4/6] spotless --- .../cluster/metadata/IndexMetadata.java | 4 +- .../DataStreamDeprecationChecker.java | 8 +-- .../DataStreamDeprecationCheckerTests.java | 58 ++++++++++++++----- .../IndexDeprecationCheckerTests.java | 29 ++++------ 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index f25031f1ce3d8..39351d5f8ecb5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -1889,7 +1889,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); - MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, - Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); - DataStream dataStream = createTestDataStream(oldIndexCount, 0, newIndexCount, 0, nameToIndexMetadata, expectedIndices, - mappingMetadata, randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)); + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + DataStream dataStream = createTestDataStream( + oldIndexCount, + 0, + newIndexCount, + 0, + nameToIndexMetadata, + expectedIndices, + mappingMetadata, + randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019) + ); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) .indices(nameToIndexMetadata) @@ -97,8 +107,8 @@ public void testOldIndicesCheckWithPercolatorField() { DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", - "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + - "fields.", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", false, ofEntries( entry("reindex_required", true), @@ -147,8 +157,10 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { Map nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); - MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, - Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); DataStream dataStream = createTestDataStream( oldOpenIndexCount, oldClosedIndexCount, @@ -190,8 +202,16 @@ private DataStream createTestDataStream( Map nameToIndexMetadata, Set expectedIndices ) { - return createTestDataStream(oldOpenIndexCount, oldClosedIndexCount, newOpenIndexCount, newClosedIndexCount, nameToIndexMetadata, - expectedIndices, MappingMetadata.EMPTY_MAPPINGS, TransportVersion.fromId(0)); + return createTestDataStream( + oldOpenIndexCount, + oldClosedIndexCount, + newOpenIndexCount, + newClosedIndexCount, + nameToIndexMetadata, + expectedIndices, + MappingMetadata.EMPTY_MAPPINGS, + TransportVersion.fromId(0) + ); } /* @@ -253,8 +273,12 @@ private Index createOldIndex( return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata, - MappingMetadata mappingMetadata) { + private Index createNewIndex( + int suffix, + boolean isClosed, + Map nameToIndexMetadata, + MappingMetadata mappingMetadata + ) { return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } @@ -371,8 +395,10 @@ public void testOldIndicesIgnoredWithPercolatorField() { Map nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); - MappingMetadata mappingMetadata = new MappingMetadata(MapperService.SINGLE_MAPPING_NAME, - Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text")))); + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); for (int i = 0; i < oldIndexCount; i++) { Settings.Builder settings = settings(IndexVersion.fromId(9000019)); @@ -424,8 +450,8 @@ public void testOldIndicesIgnoredWithPercolatorField() { DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", - "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + - "fields.", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", false, ofEntries( entry("reindex_required", true), diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 9beb45566abc8..fdf4af37fed49 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -17,7 +17,6 @@ import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.collect.ImmutableOpenMap; @@ -44,8 +43,6 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.nullValue; -import static org.mockito.ArgumentMatchers.isNull; public class IndexDeprecationCheckerTests extends ESTestCase { @@ -202,8 +199,7 @@ public void testOldIndicesCheckWithPercolatorFields() { .numberOfShards(1) .numberOfReplicas(0) .state(indexMetdataState) - .putMapping( - """ + .putMapping(""" { "properties": { "query": { @@ -214,8 +210,7 @@ public void testOldIndicesCheckWithPercolatorFields() { } } } - """ - ) + """) .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) .build(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); @@ -223,8 +218,8 @@ public void testOldIndicesCheckWithPercolatorFields() { DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", - "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + - "Field [query] is of type [_doc]", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", false, Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); @@ -244,8 +239,7 @@ public void testLatestIndicesCheckWithPercolatorFields() { .numberOfShards(1) .numberOfReplicas(0) .state(indexMetdataState) - .putMapping( - """ + .putMapping(""" { "properties": { "query": { @@ -256,8 +250,7 @@ public void testLatestIndicesCheckWithPercolatorFields() { } } } - """ - ) + """) .transportVersion(TransportVersion.current()) .build(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); @@ -490,8 +483,7 @@ public void testOldIgnoreIndicesCheckWithPercolatorFields() { .numberOfShards(1) .numberOfReplicas(0) .state(indexMetdataState) - .putMapping( - """ + .putMapping(""" { "properties": { "query": { @@ -502,8 +494,7 @@ public void testOldIgnoreIndicesCheckWithPercolatorFields() { } } } - """ - ) + """) .transportVersion(TransportVersion.fromId(9000019)) .build(); ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); @@ -511,8 +502,8 @@ public void testOldIgnoreIndicesCheckWithPercolatorFields() { DeprecationIssue.Level.CRITICAL, "Field mappings with incompatible percolator type", "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", - "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + - "Field [query] is of type [_doc]", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", false, Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); From f135b7245a5f346c5f8327249aa3d73ef304b2c5 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 14 Nov 2025 18:27:27 +0000 Subject: [PATCH 5/6] [CI] Update transport version definitions --- .../definitions/referable/index_created_transport_version.csv | 2 +- server/src/main/resources/transport/upper_bounds/8.19.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.1.csv | 2 +- server/src/main/resources/transport/upper_bounds/9.2.csv | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv index 711a96c6acee7..3571ebf30ab52 100644 --- a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -1 +1 @@ -9220000 +9220000,9185009,9112014,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index febec42efcb5b..09daf16060da0 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,8841074 +index_created_transport_version,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 6a3a21bfff8bb..37a8af514597a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -initial_9.1.8,9112013 +index_created_transport_version,9112014 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 7ebcff29ca6da..0b2d3c451ce12 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -initial_9.2.2,9185008 +index_created_transport_version,9185009 From ed05c1be7090a95433e97be78d83261a602e5671 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Fri, 14 Nov 2025 11:45:33 -0800 Subject: [PATCH 6/6] update tests --- .../cluster/reroute/ClusterRerouteResponseTests.java | 2 ++ .../org/elasticsearch/cluster/ClusterStateTests.java | 7 +++++++ .../elasticsearch/cluster/metadata/MetadataTests.java | 2 ++ .../cluster/metadata/ProjectMetadataTests.java | 10 ++++++++++ .../cluster/metadata/ToAndFromJsonMetadataTests.java | 4 ++++ 5 files changed, 25 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index a73e9635a53b3..d830e9ebbbbf8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -160,6 +160,7 @@ public void testToXContentWithDeprecatedClusterState() { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -249,6 +250,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 45db3bb22cc24..0582e66efdc54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -497,6 +497,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -529,6 +530,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "another-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -554,6 +556,7 @@ public void testToXContentWithMultipleProjects() throws IOException { }, "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1105,6 +1108,7 @@ public void testToXContent() throws IOException { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1381,6 +1385,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1663,6 +1668,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1854,6 +1860,7 @@ public void testToXContentSameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 417733c8fdbbd..43832d22372aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -828,6 +828,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { "indices": { "index-1": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -862,6 +863,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { }, "index-2": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index 366faa8817f60..a5f124731c88a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2244,6 +2244,7 @@ public void testToXContent() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2281,6 +2282,7 @@ public void testToXContent() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2320,6 +2322,7 @@ public void testToXContent() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2361,6 +2364,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2397,6 +2401,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2511,6 +2516,7 @@ public void testToXContentMultiProject() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2548,6 +2554,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2587,6 +2594,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2628,6 +2636,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2664,6 +2673,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index b133bbde7552a..e130c0b82be2f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -398,6 +398,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -565,6 +566,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -675,6 +677,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -815,6 +818,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1,