From a8c56040879ac0f4d995eee725ca7cba24d519e5 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 17 Nov 2025 12:24:30 -0800 Subject: [PATCH] Add deprecation checks for indices with percolator fields (#138118) This change modifies the deprecation API to check for any indices prior to 9.latest that have percolator fields in the mappings. Any that do are then returned as required to be reindexed or deleted. Read-only is excluded as this would not fix the underlying transport version issue. Note this change only concerns user created indices. System indices will need to be addressed separately as part of the migration API, but should be able to share code. ES-13234 --- .../cluster/metadata/IndexMetadata.java | 45 ++++ .../metadata/MetadataCreateIndexService.java | 1 + .../index_created_transport_version.csv | 1 + .../resources/transport/upper_bounds/8.19.csv | 2 +- .../resources/transport/upper_bounds/9.1.csv | 2 +- .../resources/transport/upper_bounds/9.2.csv | 2 +- .../resources/transport/upper_bounds/9.3.csv | 2 +- .../reroute/ClusterRerouteResponseTests.java | 2 + .../cluster/ClusterStateTests.java | 7 + .../cluster/metadata/IndexMetadataTests.java | 5 + .../cluster/metadata/MetadataTests.java | 2 + .../metadata/ProjectMetadataTests.java | 10 + .../metadata/ToAndFromJsonMetadataTests.java | 4 + .../deprecation/DeprecatedIndexPredicate.java | 131 ++++++++++- .../DataStreamDeprecationChecker.java | 52 ++++- .../deprecation/IndexDeprecationChecker.java | 216 ++++++++---------- .../DataStreamDeprecationCheckerTests.java | 195 ++++++++++++++-- .../IndexDeprecationCheckerTests.java | 111 +++++++++ .../AbstractWatcherIntegrationTestCase.java | 1 + 19 files changed, 649 insertions(+), 142 deletions(-) create mode 100644 server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 253440f078d44..5a8cf981462c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -153,6 +153,7 @@ public class IndexMetadata implements Diffable, ToXContentFragmen public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; private static final TransportVersion INDEX_RESHARDING_METADATA = TransportVersion.fromName("index_resharding_metadata"); + private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version"); @Nullable public String getDownsamplingInterval() { @@ -583,6 +584,7 @@ public Iterator> settings() { public static final List PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN); static final String KEY_VERSION = "version"; + static final String KEY_TRANSPORT_VERSION = "transport_version"; static final String KEY_MAPPING_VERSION = "mapping_version"; static final String KEY_SETTINGS_VERSION = "settings_version"; static final String KEY_ALIASES_VERSION = "aliases_version"; @@ -623,6 +625,7 @@ public Iterator> settings() { private final Index index; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; @@ -708,6 +711,7 @@ public Iterator> settings() { private IndexMetadata( final Index index, final long version, + final TransportVersion transportVersion, final long mappingVersion, final long settingsVersion, final long aliasesVersion, @@ -758,6 +762,7 @@ private IndexMetadata( ) { this.index = index; this.version = version; + this.transportVersion = transportVersion; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; this.mappingsUpdatedVersion = mappingsUpdatedVersion; @@ -824,6 +829,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -887,6 +893,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -958,6 +965,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1020,6 +1028,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1077,6 +1086,7 @@ public IndexMetadata withIncrementedVersion() { return new IndexMetadata( this.index, this.version + 1, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1139,6 +1149,10 @@ public long getVersion() { return this.version; } + public TransportVersion getTransportVersion() { + return transportVersion; + } + public long getMappingVersion() { return mappingVersion; } @@ -1671,6 +1685,7 @@ private static class IndexMetadataDiff implements Diff { private final String index; private final int routingNumShards; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; private final long settingsVersion; private final long aliasesVersion; @@ -1704,6 +1719,7 @@ private static class IndexMetadataDiff implements Diff { IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); version = after.version; + transportVersion = after.transportVersion; mappingVersion = after.mappingVersion; settingsVersion = after.settingsVersion; aliasesVersion = after.aliasesVersion; @@ -1757,6 +1773,9 @@ private static class IndexMetadataDiff implements Diff { index = in.readString(); routingNumShards = in.readInt(); version = in.readLong(); + transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0); mappingVersion = in.readVLong(); settingsVersion = in.readVLong(); aliasesVersion = in.readVLong(); @@ -1824,6 +1843,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeInt(routingNumShards); out.writeLong(version); + if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) { + TransportVersion.writeVersion(transportVersion, out); + } out.writeVLong(mappingVersion); out.writeVLong(settingsVersion); out.writeVLong(aliasesVersion); @@ -1864,6 +1886,7 @@ public void writeTo(StreamOutput out) throws IOException { public IndexMetadata apply(IndexMetadata part) { Builder builder = builder(index); builder.version(version); + builder.transportVersion(transportVersion); builder.mappingVersion(mappingVersion); builder.settingsVersion(settingsVersion); builder.aliasesVersion(aliasesVersion); @@ -1907,6 +1930,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { public static IndexMetadata readFrom(StreamInput in, @Nullable Function mappingLookup) throws IOException { Builder builder = new Builder(in.readString()); builder.version(in.readLong()); + builder.transportVersion( + in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0) + ); builder.mappingVersion(in.readVLong()); builder.settingsVersion(in.readVLong()); builder.aliasesVersion(in.readVLong()); @@ -1974,6 +2002,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.state(State.fromString(parser.text())); case KEY_VERSION -> builder.version(parser.longValue()); + case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text())); case KEY_MAPPING_VERSION -> { mappingVersion = true; builder.mappingVersion(parser.longValue()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 44bf6b93c014f..952b3080d3713 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -627,6 +627,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.transportVersion(TransportVersion.current()); tmpImdBuilder.system(isSystem); // Set up everything, now locally create the index to see that things are ok, and apply diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv new file mode 100644 index 0000000000000..f25ff514b4116 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -0,0 +1 @@ +9221000,9185009,9112014,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index febec42efcb5b..09daf16060da0 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,8841074 +index_created_transport_version,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 6a3a21bfff8bb..37a8af514597a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -initial_9.1.8,9112013 +index_created_transport_version,9112014 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 7ebcff29ca6da..0b2d3c451ce12 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -initial_9.2.2,9185008 +index_created_transport_version,9185009 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 61602dea24d29..cc71afa89aa4a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,9213000 +index_created_transport_version,9221000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index a73e9635a53b3..d830e9ebbbbf8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -160,6 +160,7 @@ public void testToXContentWithDeprecatedClusterState() { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -249,6 +250,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 45db3bb22cc24..0582e66efdc54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -497,6 +497,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -529,6 +530,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "another-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -554,6 +556,7 @@ public void testToXContentWithMultipleProjects() throws IOException { }, "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1105,6 +1108,7 @@ public void testToXContent() throws IOException { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1381,6 +1385,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1663,6 +1668,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1854,6 +1860,7 @@ public void testToXContentSameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d0aae463dd193..3f4410a29a137 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -151,6 +152,7 @@ public void testIndexMetadataSerialization() throws IOException { ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); @@ -178,6 +180,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata, deserialized); assertEquals(metadata.hashCode(), deserialized.hashCode()); + assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion()); @@ -214,6 +217,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -282,6 +286,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro fromXContentMeta ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 417733c8fdbbd..43832d22372aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -828,6 +828,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { "indices": { "index-1": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -862,6 +863,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { }, "index-2": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index 366faa8817f60..a5f124731c88a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2244,6 +2244,7 @@ public void testToXContent() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2281,6 +2282,7 @@ public void testToXContent() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2320,6 +2322,7 @@ public void testToXContent() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2361,6 +2364,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2397,6 +2401,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2511,6 +2516,7 @@ public void testToXContentMultiProject() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2548,6 +2554,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2587,6 +2594,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2628,6 +2636,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2664,6 +2673,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index b133bbde7552a..e130c0b82be2f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -398,6 +398,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -565,6 +566,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -675,6 +677,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -815,6 +818,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index 78a9029165817..bc63a10f0fbd1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -14,6 +15,11 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; public class DeprecatedIndexPredicate { @@ -53,7 +59,7 @@ public static Predicate getReindexRequiredPredicate( * if false, only those without a block are returned * @param includeSystem if true, all indices including system will be returned, * if false, only non-system indices are returned - * @return a predicate that returns true for indices that need to be reindexed + * @return returns true for indices that need to be reindexed */ public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) @@ -62,6 +68,59 @@ && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + /** + * This method checks if this index is on the current transport version for the current minor release version. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns true for indices that need to be reindexed + */ + public static boolean reindexRequiredForTransportVersion( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + return transportVersionBeforeCurrentMinorRelease(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) + && isNotSearchableSnapshot(indexMetadata) + && matchBlockedStatus(indexMetadata, filterToBlockedStatus); + } + + /** + * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version + * for the current minor release. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns a message as a string for each incompatible percolator field found + */ + public static List reindexRequiredForPecolatorFields( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + List percolatorIncompatibleFieldMappings = new ArrayList<>(); + if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) { + percolatorIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + indexMetadata.mapping().type(), + indexMetadata.mapping().sourceAsMap(), + property -> "percolator".equals(property.get("type")), + (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]", + "", + "" + ) + ); + } + return percolatorIncompatibleFieldMappings; + } + private static boolean isNotSystem(IndexMetadata indexMetadata) { return indexMetadata.isSystem() == false; } @@ -77,4 +136,74 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus; } + + private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { + // We divide each transport version by 1000 to get the base id. + return indexMetadata.getTransportVersion().id() / 1000 < TransportVersion.current().id() / 1000; + } + + /** + * iterates through the "properties" field of mappings and returns any predicates that match in the + * form of issue-strings. + * + * @param type the document type + * @param parentMap the mapping to read properties from + * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true + * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation + * @return a list of issues found in fields + */ + @SuppressWarnings("unchecked") + public static List findInPropertiesRecursively( + String type, + Map parentMap, + Function, Boolean> predicate, + BiFunction, String> fieldFormatter, + String fieldBeginMarker, + String fieldEndMarker + ) { + List issues = new ArrayList<>(); + Map properties = (Map) parentMap.get("properties"); + if (properties == null) { + return issues; + } + for (Map.Entry entry : properties.entrySet()) { + Map valueMap = (Map) entry.getValue(); + if (predicate.apply(valueMap)) { + issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); + } + + Map values = (Map) valueMap.get("fields"); + if (values != null) { + for (Map.Entry multifieldEntry : values.entrySet()) { + Map multifieldValueMap = (Map) multifieldEntry.getValue(); + if (predicate.apply(multifieldValueMap)) { + issues.add( + fieldBeginMarker + + fieldFormatter.apply(type, entry) + + ", multifield: " + + multifieldEntry.getKey() + + fieldEndMarker + ); + } + if (multifieldValueMap.containsKey("properties")) { + issues.addAll( + findInPropertiesRecursively( + type, + multifieldValueMap, + predicate, + fieldFormatter, + fieldBeginMarker, + fieldEndMarker + ) + ); + } + } + } + if (valueMap.containsKey("properties")) { + issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); + } + } + + return issues; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index b05bdb25df792..1c54bb07402a3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -88,9 +88,25 @@ public Map> check(ProjectMetadata project) { static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); - + Set percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, false); + if (percolatorIndicesNeedingUpgrade.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()), + entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade) + ) + ); + } Set indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, project, false); - if (indicesNeedingUpgrade.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -112,6 +128,24 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata p static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); + Set percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, true); + if (percolatorIgnoredIndices.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()), + entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices) + ) + ); + } Set ignoredIndices = getReindexRequiredIndices(backingIndices, project, true); if (ignoredIndices.isEmpty() == false) { return new DeprecationIssue( @@ -144,6 +178,20 @@ private static Set getReindexRequiredIndices( .collect(Collectors.toUnmodifiableSet()); } + private static Set getReindexRequiredIndicesWithPercolatorFields( + List backingIndices, + ProjectMetadata project, + boolean filterToBlockedStatus + ) { + return backingIndices.stream() + .filter( + index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields(project.index(index), filterToBlockedStatus, false) + .isEmpty() == false + ) + .map(Index::getName) + .collect(Collectors.toUnmodifiableSet()); + } + @Override public String getName() { return NAME; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 5f1a3310027c3..8a841afc46371 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -29,8 +29,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; @@ -93,37 +91,56 @@ private DeprecationIssue oldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - // TODO: this check needs to be revised. It's trivially true right now. - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and requires action before upgrading to %d.0. The following transforms are " - + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " - + "transforms destination indices for your upgrade.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + false, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://ela.st/es-deprecation-9-index-version", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to %d.0. The following " + + "transforms are configured to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to prepare transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://ela.st/es-deprecation-9-index-version", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -137,40 +154,60 @@ private DeprecationIssue ignoredOldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " - + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " - + "about how to handle your transforms destination indices.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + true, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", - "https://ela.st/es-deprecation-9-index-version", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in " - + Version.CURRENT.major - + ".0", + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", + "https://ela.st/es-deprecation-9-index-version", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in " + + Version.CURRENT.major + + ".0", + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -272,71 +309,6 @@ private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer findInPropertiesRecursively( - String type, - Map parentMap, - Function, Boolean> predicate, - BiFunction, String> fieldFormatter, - String fieldBeginMarker, - String fieldEndMarker - ) { - List issues = new ArrayList<>(); - Map properties = (Map) parentMap.get("properties"); - if (properties == null) { - return issues; - } - for (Map.Entry entry : properties.entrySet()) { - Map valueMap = (Map) entry.getValue(); - if (predicate.apply(valueMap)) { - issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); - } - - Map values = (Map) valueMap.get("fields"); - if (values != null) { - for (Map.Entry multifieldEntry : values.entrySet()) { - Map multifieldValueMap = (Map) multifieldEntry.getValue(); - if (predicate.apply(multifieldValueMap)) { - issues.add( - fieldBeginMarker - + fieldFormatter.apply(type, entry) - + ", multifield: " - + multifieldEntry.getKey() - + fieldEndMarker - ); - } - if (multifieldValueMap.containsKey("properties")) { - issues.addAll( - findInPropertiesRecursively( - type, - multifieldValueMap, - predicate, - fieldFormatter, - fieldBeginMarker, - fieldEndMarker - ) - ); - } - } - } - if (valueMap.containsKey("properties")) { - issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); - } - } - - return issues; - } - private DeprecationIssue deprecatedCamelCasePattern( IndexMetadata indexMetadata, ProjectMetadata project, @@ -346,7 +318,7 @@ private DeprecationIssue deprecatedCamelCasePattern( fieldLevelMappingIssue( indexMetadata, ((mappingMetadata, sourceAsMap) -> fields.addAll( - findInPropertiesRecursively( + DeprecatedIndexPredicate.findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, this::isDateFieldWithCamelCasePattern, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 49611209862a3..a858b0f7a3b69 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -7,16 +7,19 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; @@ -73,6 +76,55 @@ public void testOldIndicesCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesCheckWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + DataStream dataStream = createTestDataStream( + oldIndexCount, + 0, + newIndexCount, + 0, + nameToIndexMetadata, + expectedIndices, + mappingMetadata, + randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019) + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("indices_requiring_upgrade_count", expectedIndices.size()), + entry("indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.size(), equalTo(1)); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldIndicesCheckWithOnlyNewIndices() { // This tests what happens when any old indices that we have are closed. We expect no deprecation warning. int newOpenIndexCount = randomIntBetween(1, 100); @@ -105,6 +157,10 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { Map nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); DataStream dataStream = createTestDataStream( oldOpenIndexCount, oldClosedIndexCount, @@ -138,6 +194,26 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + private DataStream createTestDataStream( + int oldOpenIndexCount, + int oldClosedIndexCount, + int newOpenIndexCount, + int newClosedIndexCount, + Map nameToIndexMetadata, + Set expectedIndices + ) { + return createTestDataStream( + oldOpenIndexCount, + oldClosedIndexCount, + newOpenIndexCount, + newClosedIndexCount, + nameToIndexMetadata, + expectedIndices, + MappingMetadata.EMPTY_MAPPINGS, + TransportVersion.fromId(0) + ); + } + /* * This creates a test DataStream with the given counts. The nameToIndexMetadata Map and the expectedIndices Set are mutable collections * that will be populated by this method. @@ -148,21 +224,23 @@ private DataStream createTestDataStream( int newOpenIndexCount, int newClosedIndexCount, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion oldTransportVersion ) { List allIndices = new ArrayList<>(); for (int i = 0; i < oldOpenIndexCount; i++) { - allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < oldClosedIndexCount; i++) { - allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < newOpenIndexCount; i++) { - allIndices.add(createNewIndex(i, false, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, false, nameToIndexMetadata, mappingMetadata)); } for (int i = 0; i < newClosedIndexCount; i++) { - allIndices.add(createNewIndex(i, true, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, true, nameToIndexMetadata, mappingMetadata)); } DataStream dataStream = new DataStream( @@ -188,13 +266,20 @@ private Index createOldIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { - return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices); + return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata) { - return createIndex(false, suffix, isClosed, nameToIndexMetadata, null); + private Index createNewIndex( + int suffix, + boolean isClosed, + Map nameToIndexMetadata, + MappingMetadata mappingMetadata + ) { + return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } private Index createIndex( @@ -202,7 +287,9 @@ private Index createIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { Settings.Builder settingsBuilder = isOld ? settings(IndexVersion.fromId(7170099)) : settings(IndexVersion.current()); String indexName = (isOld ? "old-" : "new-") + (isClosed ? "closed-" : "") + "data-stream-index-" + suffix; @@ -216,7 +303,9 @@ private Index createIndex( IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(settingsBuilder) .numberOfShards(1) - .numberOfReplicas(0); + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(transportVersion); if (isClosed) { indexMetadataBuilder.state(IndexMetadata.State.CLOSE); } @@ -251,7 +340,7 @@ public void testOldIndicesIgnoredWarningCheck() { } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } @@ -298,6 +387,86 @@ public void testOldIndicesIgnoredWarningCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesIgnoredWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(9000019)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + expectedIndices.add(indexName); + + Settings.Builder settingsBuilder = settings; + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + + for (int i = 0; i < newIndexCount; i++) { + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, mappingMetadata); + allIndices.add(newIndex); + } + + DataStream dataStream = new DataStream( + randomAlphaOfLength(10), + allIndices, + randomNegativeLong(), + Map.of(), + randomBoolean(), + false, + false, + randomBoolean(), + randomFrom(IndexMode.values()), + null, + randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null), + List.of(), + randomBoolean(), + null + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()), + entry("ignored_indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldSystemDataStreamIgnored() { // We do not want system data streams coming back in the deprecation info API int oldIndexCount = randomIntBetween(1, 100); @@ -319,7 +488,7 @@ public void testOldSystemDataStreamIgnored() { nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } DataStream dataStream = new DataStream( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 5ea8a9eea6b8e..fdf4af37fed49 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamMetadata; @@ -191,6 +192,76 @@ public void testMultipleOldIndicesCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + + public void testLatestIndicesCheckWithPercolatorFields() { + IndexVersion latestVersion = IndexVersion.current(); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(latestVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.current()) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex.isEmpty(), equalTo(true)); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -405,6 +476,46 @@ public void testMultipleOldIndicesIgnoredCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIgnoreIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index d237e08b30c19..47215174d0639 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -275,6 +275,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia newSettings.remove("index.uuid"); newSettings.remove("index.creation_date"); newSettings.remove("index.version.created"); + newSettings.remove("index.transport_version.created"); assertAcked(indicesAdmin().prepareCreate(to).setMapping(mapping.sourceAsMap()).setSettings(newSettings)); ensureGreen(to);