diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java index 253440f078d44..5a8cf981462c8 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadata.java @@ -153,6 +153,7 @@ public class IndexMetadata implements Diffable, ToXContentFragmen public static final String EVENT_INGESTED_FIELD_NAME = "event.ingested"; private static final TransportVersion INDEX_RESHARDING_METADATA = TransportVersion.fromName("index_resharding_metadata"); + private static final TransportVersion INDEX_CREATED_TRANSPORT_VERSION = TransportVersion.fromName("index_created_transport_version"); @Nullable public String getDownsamplingInterval() { @@ -583,6 +584,7 @@ public Iterator> settings() { public static final List PARTIALLY_MOUNTED_INDEX_TIER_PREFERENCE = List.of(DataTier.DATA_FROZEN); static final String KEY_VERSION = "version"; + static final String KEY_TRANSPORT_VERSION = "transport_version"; static final String KEY_MAPPING_VERSION = "mapping_version"; static final String KEY_SETTINGS_VERSION = "settings_version"; static final String KEY_ALIASES_VERSION = "aliases_version"; @@ -623,6 +625,7 @@ public Iterator> settings() { private final Index index; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; @@ -708,6 +711,7 @@ public Iterator> settings() { private IndexMetadata( final Index index, final long version, + final TransportVersion transportVersion, final long mappingVersion, final long settingsVersion, final long aliasesVersion, @@ -758,6 +762,7 @@ private IndexMetadata( ) { this.index = index; this.version = version; + this.transportVersion = transportVersion; assert mappingVersion >= 0 : mappingVersion; this.mappingVersion = mappingVersion; this.mappingsUpdatedVersion = mappingsUpdatedVersion; @@ -824,6 +829,7 @@ IndexMetadata withMappingMetadata(MappingMetadata mapping) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -887,6 +893,7 @@ public IndexMetadata withInSyncAllocationIds(int shardId, Set inSyncSet) return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -958,6 +965,7 @@ public IndexMetadata withSetPrimaryTerm(int shardId, long primaryTerm) { return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1020,6 +1028,7 @@ public IndexMetadata withTimestampRanges(IndexLongFieldRange timestampRange, Ind return new IndexMetadata( this.index, this.version, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1077,6 +1086,7 @@ public IndexMetadata withIncrementedVersion() { return new IndexMetadata( this.index, this.version + 1, + this.transportVersion, this.mappingVersion, this.settingsVersion, this.aliasesVersion, @@ -1139,6 +1149,10 @@ public long getVersion() { return this.version; } + public TransportVersion getTransportVersion() { + return transportVersion; + } + public long getMappingVersion() { return mappingVersion; } @@ -1671,6 +1685,7 @@ private static class IndexMetadataDiff implements Diff { private final String index; private final int routingNumShards; private final long version; + private final TransportVersion transportVersion; private final long mappingVersion; private final long settingsVersion; private final long aliasesVersion; @@ -1704,6 +1719,7 @@ private static class IndexMetadataDiff implements Diff { IndexMetadataDiff(IndexMetadata before, IndexMetadata after) { index = after.index.getName(); version = after.version; + transportVersion = after.transportVersion; mappingVersion = after.mappingVersion; settingsVersion = after.settingsVersion; aliasesVersion = after.aliasesVersion; @@ -1757,6 +1773,9 @@ private static class IndexMetadataDiff implements Diff { index = in.readString(); routingNumShards = in.readInt(); version = in.readLong(); + transportVersion = in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0); mappingVersion = in.readVLong(); settingsVersion = in.readVLong(); aliasesVersion = in.readVLong(); @@ -1824,6 +1843,9 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeInt(routingNumShards); out.writeLong(version); + if (out.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION)) { + TransportVersion.writeVersion(transportVersion, out); + } out.writeVLong(mappingVersion); out.writeVLong(settingsVersion); out.writeVLong(aliasesVersion); @@ -1864,6 +1886,7 @@ public void writeTo(StreamOutput out) throws IOException { public IndexMetadata apply(IndexMetadata part) { Builder builder = builder(index); builder.version(version); + builder.transportVersion(transportVersion); builder.mappingVersion(mappingVersion); builder.settingsVersion(settingsVersion); builder.aliasesVersion(aliasesVersion); @@ -1907,6 +1930,11 @@ public static IndexMetadata readFrom(StreamInput in) throws IOException { public static IndexMetadata readFrom(StreamInput in, @Nullable Function mappingLookup) throws IOException { Builder builder = new Builder(in.readString()); builder.version(in.readLong()); + builder.transportVersion( + in.getTransportVersion().supports(INDEX_CREATED_TRANSPORT_VERSION) + ? TransportVersion.readVersion(in) + : TransportVersion.fromId(0) + ); builder.mappingVersion(in.readVLong()); builder.settingsVersion(in.readVLong()); builder.aliasesVersion(in.readVLong()); @@ -1974,6 +2002,9 @@ public static IndexMetadata readFrom(StreamInput in, @Nullable Function builder.state(State.fromString(parser.text())); case KEY_VERSION -> builder.version(parser.longValue()); + case KEY_TRANSPORT_VERSION -> builder.transportVersion(TransportVersion.fromString(parser.text())); case KEY_MAPPING_VERSION -> { mappingVersion = true; builder.mappingVersion(parser.longValue()); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java index 44bf6b93c014f..952b3080d3713 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java @@ -627,6 +627,7 @@ private IndexMetadata buildAndValidateTemporaryIndexMetadata( final IndexMetadata.Builder tmpImdBuilder = IndexMetadata.builder(request.index()); tmpImdBuilder.setRoutingNumShards(routingNumShards); tmpImdBuilder.settings(indexSettings); + tmpImdBuilder.transportVersion(TransportVersion.current()); tmpImdBuilder.system(isSystem); // Set up everything, now locally create the index to see that things are ok, and apply diff --git a/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv new file mode 100644 index 0000000000000..f25ff514b4116 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/index_created_transport_version.csv @@ -0,0 +1 @@ +9221000,9185009,9112014,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/8.19.csv b/server/src/main/resources/transport/upper_bounds/8.19.csv index febec42efcb5b..09daf16060da0 100644 --- a/server/src/main/resources/transport/upper_bounds/8.19.csv +++ b/server/src/main/resources/transport/upper_bounds/8.19.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,8841074 +index_created_transport_version,8841075 diff --git a/server/src/main/resources/transport/upper_bounds/9.1.csv b/server/src/main/resources/transport/upper_bounds/9.1.csv index 6a3a21bfff8bb..37a8af514597a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.1.csv +++ b/server/src/main/resources/transport/upper_bounds/9.1.csv @@ -1 +1 @@ -initial_9.1.8,9112013 +index_created_transport_version,9112014 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 7ebcff29ca6da..0b2d3c451ce12 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -initial_9.2.2,9185008 +index_created_transport_version,9185009 diff --git a/server/src/main/resources/transport/upper_bounds/9.3.csv b/server/src/main/resources/transport/upper_bounds/9.3.csv index 61602dea24d29..cc71afa89aa4a 100644 --- a/server/src/main/resources/transport/upper_bounds/9.3.csv +++ b/server/src/main/resources/transport/upper_bounds/9.3.csv @@ -1 +1 @@ -batched_response_might_include_reduction_failure,9213000 +index_created_transport_version,9221000 diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java index a73e9635a53b3..d830e9ebbbbf8 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/reroute/ClusterRerouteResponseTests.java @@ -160,6 +160,7 @@ public void testToXContentWithDeprecatedClusterState() { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -249,6 +250,7 @@ public void testToXContentWithDeprecatedClusterStateAndMetadata() { "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java index 45db3bb22cc24..0582e66efdc54 100644 --- a/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/ClusterStateTests.java @@ -497,6 +497,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -529,6 +530,7 @@ public void testToXContentWithMultipleProjects() throws IOException { "indices": { "another-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -554,6 +556,7 @@ public void testToXContentWithMultipleProjects() throws IOException { }, "common-index": { "version": 2, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1105,6 +1108,7 @@ public void testToXContent() throws IOException { "indices": { "index": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -1381,6 +1385,7 @@ public void testToXContent_FlatSettingTrue_ReduceMappingFalse() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1663,6 +1668,7 @@ public void testToXContent_FlatSettingFalse_ReduceMappingTrue() throws IOExcepti "indices" : { "index" : { "version" : 1, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -1854,6 +1860,7 @@ public void testToXContentSameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java index d0aae463dd193..3f4410a29a137 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataTests.java @@ -100,6 +100,7 @@ public void testIndexMetadataSerialization() throws IOException { IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -151,6 +152,7 @@ public void testIndexMetadataSerialization() throws IOException { ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); @@ -178,6 +180,7 @@ public void testIndexMetadataSerialization() throws IOException { assertEquals(metadata, deserialized); assertEquals(metadata.hashCode(), deserialized.hashCode()); + assertEquals(metadata.getTransportVersion(), deserialized.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), deserialized.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), deserialized.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), deserialized.getCreationVersion()); @@ -214,6 +217,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro IndexMetadata metadata = IndexMetadata.builder("foo") .settings(indexSettings(numShard, numberOfReplicas).put("index.version.created", 1)) + .transportVersion(TransportVersion.current()) .creationDate(randomLong()) .primaryTerm(0, 2) .setRoutingNumShards(32) @@ -282,6 +286,7 @@ public void testIndexMetadataFromXContentParsingWithoutEventIngestedField() thro fromXContentMeta ); assertEquals(metadata.hashCode(), fromXContentMeta.hashCode()); + assertEquals(metadata.getTransportVersion(), fromXContentMeta.getTransportVersion()); assertEquals(metadata.getNumberOfReplicas(), fromXContentMeta.getNumberOfReplicas()); assertEquals(metadata.getNumberOfShards(), fromXContentMeta.getNumberOfShards()); assertEquals(metadata.getCreationVersion(), fromXContentMeta.getCreationVersion()); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java index 417733c8fdbbd..43832d22372aa 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataTests.java @@ -828,6 +828,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { "indices": { "index-1": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -862,6 +863,7 @@ public void testSingleNonDefaultProjectXContent() throws IOException { }, "index-2": { "version": 1, + "transport_version": "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java index 366faa8817f60..a5f124731c88a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ProjectMetadataTests.java @@ -2244,6 +2244,7 @@ public void testToXContent() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2281,6 +2282,7 @@ public void testToXContent() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2320,6 +2322,7 @@ public void testToXContent() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2361,6 +2364,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2397,6 +2401,7 @@ public void testToXContent() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2511,6 +2516,7 @@ public void testToXContentMultiProject() throws IOException { "indices": { "index-01": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2548,6 +2554,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-02": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2587,6 +2594,7 @@ public void testToXContentMultiProject() throws IOException { }, "index-03": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2628,6 +2636,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000001": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, @@ -2664,6 +2673,7 @@ public void testToXContentMultiProject() throws IOException { }, ".ds-logs-ultron-2024.08.30-000002": { "version": 1, + "transport_version" : "0", "mapping_version": 1, "settings_version": 1, "aliases_version": 1, diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java index b133bbde7552a..e130c0b82be2f 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/ToAndFromJsonMetadataTests.java @@ -398,6 +398,7 @@ public void testToXContentAPI_SameTypeName() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -565,6 +566,7 @@ public void testToXContentAPI_FlatSettingTrue_ReduceMappingFalse() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -675,6 +677,7 @@ public void testToXContentAPI_FlatSettingFalse_ReduceMappingTrue() throws IOExce "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, @@ -815,6 +818,7 @@ public void testToXContentAPIReservedMetadata() throws IOException { "indices" : { "index" : { "version" : 2, + "transport_version" : "0", "mapping_version" : 1, "settings_version" : 1, "aliases_version" : 1, diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java index 78a9029165817..bc63a10f0fbd1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecatedIndexPredicate.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.core.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; @@ -14,6 +15,11 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; +import java.util.function.Function; import java.util.function.Predicate; public class DeprecatedIndexPredicate { @@ -53,7 +59,7 @@ public static Predicate getReindexRequiredPredicate( * if false, only those without a block are returned * @param includeSystem if true, all indices including system will be returned, * if false, only non-system indices are returned - * @return a predicate that returns true for indices that need to be reindexed + * @return returns true for indices that need to be reindexed */ public static boolean reindexRequired(IndexMetadata indexMetadata, boolean filterToBlockedStatus, boolean includeSystem) { return creationVersionBeforeMinimumWritableVersion(indexMetadata) @@ -62,6 +68,59 @@ && isNotSearchableSnapshot(indexMetadata) && matchBlockedStatus(indexMetadata, filterToBlockedStatus); } + /** + * This method checks if this index is on the current transport version for the current minor release version. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns true for indices that need to be reindexed + */ + public static boolean reindexRequiredForTransportVersion( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + return transportVersionBeforeCurrentMinorRelease(indexMetadata) + && (includeSystem || isNotSystem(indexMetadata)) + && isNotSearchableSnapshot(indexMetadata) + && matchBlockedStatus(indexMetadata, filterToBlockedStatus); + } + + /** + * This method checks if this index requires reindexing based on if it has percolator fields older than the current transport version + * for the current minor release. + * + * @param indexMetadata the index metadata + * @param filterToBlockedStatus if true, only indices that are write blocked will be returned, + * if false, only those without a block are returned + * @param includeSystem if true, all indices including system will be returned, + * if false, only non-system indices are returned + * @return returns a message as a string for each incompatible percolator field found + */ + public static List reindexRequiredForPecolatorFields( + IndexMetadata indexMetadata, + boolean filterToBlockedStatus, + boolean includeSystem + ) { + List percolatorIncompatibleFieldMappings = new ArrayList<>(); + if (reindexRequiredForTransportVersion(indexMetadata, filterToBlockedStatus, includeSystem) && indexMetadata.mapping() != null) { + percolatorIncompatibleFieldMappings.addAll( + findInPropertiesRecursively( + indexMetadata.mapping().type(), + indexMetadata.mapping().sourceAsMap(), + property -> "percolator".equals(property.get("type")), + (type, entry) -> "Field [" + entry.getKey() + "] is of type [" + indexMetadata.mapping().type() + "]", + "", + "" + ) + ); + } + return percolatorIncompatibleFieldMappings; + } + private static boolean isNotSystem(IndexMetadata indexMetadata) { return indexMetadata.isSystem() == false; } @@ -77,4 +136,74 @@ private static boolean creationVersionBeforeMinimumWritableVersion(IndexMetadata private static boolean matchBlockedStatus(IndexMetadata indexMetadata, boolean filterToBlockedStatus) { return MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.get(indexMetadata.getSettings()) == filterToBlockedStatus; } + + private static boolean transportVersionBeforeCurrentMinorRelease(IndexMetadata indexMetadata) { + // We divide each transport version by 1000 to get the base id. + return indexMetadata.getTransportVersion().id() / 1000 < TransportVersion.current().id() / 1000; + } + + /** + * iterates through the "properties" field of mappings and returns any predicates that match in the + * form of issue-strings. + * + * @param type the document type + * @param parentMap the mapping to read properties from + * @param predicate the predicate to check against for issues, issue is returned if predicate evaluates to true + * @param fieldFormatter a function that takes a type and mapping field entry and returns a formatted field representation + * @return a list of issues found in fields + */ + @SuppressWarnings("unchecked") + public static List findInPropertiesRecursively( + String type, + Map parentMap, + Function, Boolean> predicate, + BiFunction, String> fieldFormatter, + String fieldBeginMarker, + String fieldEndMarker + ) { + List issues = new ArrayList<>(); + Map properties = (Map) parentMap.get("properties"); + if (properties == null) { + return issues; + } + for (Map.Entry entry : properties.entrySet()) { + Map valueMap = (Map) entry.getValue(); + if (predicate.apply(valueMap)) { + issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); + } + + Map values = (Map) valueMap.get("fields"); + if (values != null) { + for (Map.Entry multifieldEntry : values.entrySet()) { + Map multifieldValueMap = (Map) multifieldEntry.getValue(); + if (predicate.apply(multifieldValueMap)) { + issues.add( + fieldBeginMarker + + fieldFormatter.apply(type, entry) + + ", multifield: " + + multifieldEntry.getKey() + + fieldEndMarker + ); + } + if (multifieldValueMap.containsKey("properties")) { + issues.addAll( + findInPropertiesRecursively( + type, + multifieldValueMap, + predicate, + fieldFormatter, + fieldBeginMarker, + fieldEndMarker + ) + ); + } + } + } + if (valueMap.containsKey("properties")) { + issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); + } + } + + return issues; + } } diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java index b05bdb25df792..1c54bb07402a3 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java @@ -88,9 +88,25 @@ public Map> check(ProjectMetadata project) { static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); - + Set percolatorIndicesNeedingUpgrade = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, false); + if (percolatorIndicesNeedingUpgrade.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("indices_requiring_upgrade_count", percolatorIndicesNeedingUpgrade.size()), + entry("indices_requiring_upgrade", percolatorIndicesNeedingUpgrade) + ) + ); + } Set indicesNeedingUpgrade = getReindexRequiredIndices(backingIndices, project, false); - if (indicesNeedingUpgrade.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, @@ -112,6 +128,24 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ProjectMetadata p static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ProjectMetadata project) { List backingIndices = dataStream.getIndices(); + Set percolatorIgnoredIndices = getReindexRequiredIndicesWithPercolatorFields(backingIndices, project, true); + if (percolatorIgnoredIndices.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", backingIndices.size()), + entry("ignored_indices_requiring_upgrade_count", percolatorIgnoredIndices.size()), + entry("ignored_indices_requiring_upgrade", percolatorIgnoredIndices) + ) + ); + } Set ignoredIndices = getReindexRequiredIndices(backingIndices, project, true); if (ignoredIndices.isEmpty() == false) { return new DeprecationIssue( @@ -144,6 +178,20 @@ private static Set getReindexRequiredIndices( .collect(Collectors.toUnmodifiableSet()); } + private static Set getReindexRequiredIndicesWithPercolatorFields( + List backingIndices, + ProjectMetadata project, + boolean filterToBlockedStatus + ) { + return backingIndices.stream() + .filter( + index -> DeprecatedIndexPredicate.reindexRequiredForPecolatorFields(project.index(index), filterToBlockedStatus, false) + .isEmpty() == false + ) + .map(Index::getName) + .collect(Collectors.toUnmodifiableSet()); + } + @Override public String getName() { return NAME; diff --git a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java index 5f1a3310027c3..8a841afc46371 100644 --- a/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java +++ b/x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/IndexDeprecationChecker.java @@ -29,8 +29,6 @@ import java.util.Map; import java.util.Objects; import java.util.function.BiConsumer; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.stream.Collectors; import static org.elasticsearch.xpack.deprecation.LegacyTiersDetection.DEPRECATION_COMMON_DETAIL; @@ -93,37 +91,56 @@ private DeprecationIssue oldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - // TODO: this check needs to be revised. It's trivially true right now. - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.CRITICAL, - "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and requires action before upgrading to %d.0. The following transforms are " - + "configured to write to this index: [%s]. Refer to the migration guide to learn more about how to prepare " - + "transforms destination indices for your upgrade.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + false, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( DeprecationIssue.Level.CRITICAL, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://ela.st/es-deprecation-9-index-version", - "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, false, false)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "One or more Transforms write to this index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and requires action before upgrading to %d.0. The following " + + "transforms are configured to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to prepare transforms destination indices for your upgrade.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://ela.st/es-deprecation-9-index-version", + "This index has version: " + currentCompatibilityVersion.toReleaseVersion(), + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -137,40 +154,60 @@ private DeprecationIssue ignoredOldIndicesCheck( ProjectMetadata project, Map> indexToTransformIds ) { - IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); // We intentionally exclude indices that are in data streams because they will be picked up by DataStreamDeprecationChecks - if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { - var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); - if (transforms.isEmpty() == false) { - return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", - "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", - Strings.format( - "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " - + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " - + "about how to handle your transforms destination indices.", - currentCompatibilityVersion.toReleaseVersion(), - Version.CURRENT.major, - String.join(", ", transforms) - ), - false, - Map.of("reindex_required", true, "transform_ids", transforms) - ); - } else { + if (isNotDataStreamIndex(indexMetadata, project)) { + // We check for percolator indices first as that will include potentially older indices as well. + List percolatorIncompatibleFieldMappings = DeprecatedIndexPredicate.reindexRequiredForPecolatorFields( + indexMetadata, + true, + false + ); + if (percolatorIncompatibleFieldMappings.isEmpty() == false) { return new DeprecationIssue( - DeprecationIssue.Level.WARNING, - "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", - "https://ela.st/es-deprecation-9-index-version", - "This read-only index has version: " - + currentCompatibilityVersion.toReleaseVersion() - + " and will be supported as read-only in " - + Version.CURRENT.major - + ".0", + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields. " + + String.join(", ", percolatorIncompatibleFieldMappings), false, - Map.of("reindex_required", true) + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) ); } + if (DeprecatedIndexPredicate.reindexRequired(indexMetadata, true, false) && isNotDataStreamIndex(indexMetadata, project)) { + IndexVersion currentCompatibilityVersion = indexMetadata.getCompatibilityVersion(); + var transforms = transformIdsForIndex(indexMetadata, indexToTransformIds); + if (transforms.isEmpty() == false) { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "One or more Transforms write to this old index with a compatibility version < " + Version.CURRENT.major + ".0", + "https://www.elastic.co/docs/deploy-manage/upgrade/prepare-to-upgrade#transform-migration", + Strings.format( + "This index was created in version [%s] and will be supported as a read-only index in %d.0. The following " + + "transforms are no longer able to write to this index: [%s]. Refer to the migration guide to learn more " + + "about how to handle your transforms destination indices.", + currentCompatibilityVersion.toReleaseVersion(), + Version.CURRENT.major, + String.join(", ", transforms) + ), + false, + Map.of("reindex_required", true, "transform_ids", transforms) + ); + } else { + return new DeprecationIssue( + DeprecationIssue.Level.WARNING, + "Old index with a compatibility version < " + Version.CURRENT.major + ".0 has been ignored", + "https://ela.st/es-deprecation-9-index-version", + "This read-only index has version: " + + currentCompatibilityVersion.toReleaseVersion() + + " and will be supported as read-only in " + + Version.CURRENT.major + + ".0", + false, + Map.of("reindex_required", true) + ); + } + } } return null; } @@ -272,71 +309,6 @@ private void fieldLevelMappingIssue(IndexMetadata indexMetadata, BiConsumer findInPropertiesRecursively( - String type, - Map parentMap, - Function, Boolean> predicate, - BiFunction, String> fieldFormatter, - String fieldBeginMarker, - String fieldEndMarker - ) { - List issues = new ArrayList<>(); - Map properties = (Map) parentMap.get("properties"); - if (properties == null) { - return issues; - } - for (Map.Entry entry : properties.entrySet()) { - Map valueMap = (Map) entry.getValue(); - if (predicate.apply(valueMap)) { - issues.add(fieldBeginMarker + fieldFormatter.apply(type, entry) + fieldEndMarker); - } - - Map values = (Map) valueMap.get("fields"); - if (values != null) { - for (Map.Entry multifieldEntry : values.entrySet()) { - Map multifieldValueMap = (Map) multifieldEntry.getValue(); - if (predicate.apply(multifieldValueMap)) { - issues.add( - fieldBeginMarker - + fieldFormatter.apply(type, entry) - + ", multifield: " - + multifieldEntry.getKey() - + fieldEndMarker - ); - } - if (multifieldValueMap.containsKey("properties")) { - issues.addAll( - findInPropertiesRecursively( - type, - multifieldValueMap, - predicate, - fieldFormatter, - fieldBeginMarker, - fieldEndMarker - ) - ); - } - } - } - if (valueMap.containsKey("properties")) { - issues.addAll(findInPropertiesRecursively(type, valueMap, predicate, fieldFormatter, fieldBeginMarker, fieldEndMarker)); - } - } - - return issues; - } - private DeprecationIssue deprecatedCamelCasePattern( IndexMetadata indexMetadata, ProjectMetadata project, @@ -346,7 +318,7 @@ private DeprecationIssue deprecatedCamelCasePattern( fieldLevelMappingIssue( indexMetadata, ((mappingMetadata, sourceAsMap) -> fields.addAll( - findInPropertiesRecursively( + DeprecatedIndexPredicate.findInPropertiesRecursively( mappingMetadata.type(), sourceAsMap, this::isDateFieldWithCamelCasePattern, diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java index 49611209862a3..a858b0f7a3b69 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationCheckerTests.java @@ -7,16 +7,19 @@ package org.elasticsearch.xpack.deprecation; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamOptions; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.cluster.metadata.MetadataIndexStateService; import org.elasticsearch.cluster.metadata.ProjectMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.Index; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.TestIndexNameExpressionResolver; import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; @@ -73,6 +76,55 @@ public void testOldIndicesCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesCheckWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + DataStream dataStream = createTestDataStream( + oldIndexCount, + 0, + newIndexCount, + 0, + nameToIndexMetadata, + expectedIndices, + mappingMetadata, + randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019) + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("indices_requiring_upgrade_count", expectedIndices.size()), + entry("indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.size(), equalTo(1)); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldIndicesCheckWithOnlyNewIndices() { // This tests what happens when any old indices that we have are closed. We expect no deprecation warning. int newOpenIndexCount = randomIntBetween(1, 100); @@ -105,6 +157,10 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { Map nameToIndexMetadata = new HashMap<>(); Set expectedIndices = new HashSet<>(); + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); DataStream dataStream = createTestDataStream( oldOpenIndexCount, oldClosedIndexCount, @@ -138,6 +194,26 @@ public void testOldIndicesCheckWithClosedAndOpenIndices() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + private DataStream createTestDataStream( + int oldOpenIndexCount, + int oldClosedIndexCount, + int newOpenIndexCount, + int newClosedIndexCount, + Map nameToIndexMetadata, + Set expectedIndices + ) { + return createTestDataStream( + oldOpenIndexCount, + oldClosedIndexCount, + newOpenIndexCount, + newClosedIndexCount, + nameToIndexMetadata, + expectedIndices, + MappingMetadata.EMPTY_MAPPINGS, + TransportVersion.fromId(0) + ); + } + /* * This creates a test DataStream with the given counts. The nameToIndexMetadata Map and the expectedIndices Set are mutable collections * that will be populated by this method. @@ -148,21 +224,23 @@ private DataStream createTestDataStream( int newOpenIndexCount, int newClosedIndexCount, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion oldTransportVersion ) { List allIndices = new ArrayList<>(); for (int i = 0; i < oldOpenIndexCount; i++) { - allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, false, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < oldClosedIndexCount; i++) { - allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices)); + allIndices.add(createOldIndex(i, true, nameToIndexMetadata, expectedIndices, mappingMetadata, oldTransportVersion)); } for (int i = 0; i < newOpenIndexCount; i++) { - allIndices.add(createNewIndex(i, false, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, false, nameToIndexMetadata, mappingMetadata)); } for (int i = 0; i < newClosedIndexCount; i++) { - allIndices.add(createNewIndex(i, true, nameToIndexMetadata)); + allIndices.add(createNewIndex(i, true, nameToIndexMetadata, mappingMetadata)); } DataStream dataStream = new DataStream( @@ -188,13 +266,20 @@ private Index createOldIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { - return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices); + return createIndex(true, suffix, isClosed, nameToIndexMetadata, expectedIndices, mappingMetadata, transportVersion); } - private Index createNewIndex(int suffix, boolean isClosed, Map nameToIndexMetadata) { - return createIndex(false, suffix, isClosed, nameToIndexMetadata, null); + private Index createNewIndex( + int suffix, + boolean isClosed, + Map nameToIndexMetadata, + MappingMetadata mappingMetadata + ) { + return createIndex(false, suffix, isClosed, nameToIndexMetadata, null, mappingMetadata, TransportVersion.current()); } private Index createIndex( @@ -202,7 +287,9 @@ private Index createIndex( int suffix, boolean isClosed, Map nameToIndexMetadata, - Set expectedIndices + Set expectedIndices, + MappingMetadata mappingMetadata, + TransportVersion transportVersion ) { Settings.Builder settingsBuilder = isOld ? settings(IndexVersion.fromId(7170099)) : settings(IndexVersion.current()); String indexName = (isOld ? "old-" : "new-") + (isClosed ? "closed-" : "") + "data-stream-index-" + suffix; @@ -216,7 +303,9 @@ private Index createIndex( IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(indexName) .settings(settingsBuilder) .numberOfShards(1) - .numberOfReplicas(0); + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(transportVersion); if (isClosed) { indexMetadataBuilder.state(IndexMetadata.State.CLOSE); } @@ -251,7 +340,7 @@ public void testOldIndicesIgnoredWarningCheck() { } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } @@ -298,6 +387,86 @@ public void testOldIndicesIgnoredWarningCheck() { assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); } + public void testOldIndicesIgnoredWithPercolatorField() { + int oldIndexCount = randomIntBetween(1, 100); + int newIndexCount = randomIntBetween(1, 100); + + List allIndices = new ArrayList<>(); + Map nameToIndexMetadata = new HashMap<>(); + Set expectedIndices = new HashSet<>(); + + MappingMetadata mappingMetadata = new MappingMetadata( + MapperService.SINGLE_MAPPING_NAME, + Map.of("properties", Map.of("query", Map.of("type", "percolator"), "field", Map.of("type", "text"))) + ); + + for (int i = 0; i < oldIndexCount; i++) { + Settings.Builder settings = settings(IndexVersion.fromId(9000019)); + + String indexName = "old-data-stream-index-" + i; + settings.put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true); + expectedIndices.add(indexName); + + Settings.Builder settingsBuilder = settings; + IndexMetadata oldIndexMetadata = IndexMetadata.builder(indexName) + .settings(settingsBuilder) + .numberOfShards(1) + .numberOfReplicas(0) + .putMapping(mappingMetadata) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + allIndices.add(oldIndexMetadata.getIndex()); + nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); + } + + for (int i = 0; i < newIndexCount; i++) { + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, mappingMetadata); + allIndices.add(newIndex); + } + + DataStream dataStream = new DataStream( + randomAlphaOfLength(10), + allIndices, + randomNegativeLong(), + Map.of(), + randomBoolean(), + false, + false, + randomBoolean(), + randomFrom(IndexMode.values()), + null, + randomFrom(DataStreamOptions.EMPTY, DataStreamOptions.FAILURE_STORE_DISABLED, DataStreamOptions.FAILURE_STORE_ENABLED, null), + List.of(), + randomBoolean(), + null + ); + + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()) + .indices(nameToIndexMetadata) + .dataStreams(Map.of(dataStream.getName(), dataStream), Map.of()) + .build(); + + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The data stream was created before 9.latest and contains mappings that must be reindexed due to containing percolator " + + "fields.", + false, + ofEntries( + entry("reindex_required", true), + entry("excluded_actions", List.of("readOnly")), + entry("total_backing_indices", oldIndexCount + newIndexCount), + entry("ignored_indices_requiring_upgrade_count", expectedIndices.size()), + entry("ignored_indices_requiring_upgrade", expectedIndices) + ) + ); + + Map> issuesByDataStream = checker.check(project); + assertThat(issuesByDataStream.containsKey(dataStream.getName()), equalTo(true)); + assertThat(issuesByDataStream.get(dataStream.getName()), equalTo(List.of(expected))); + } + public void testOldSystemDataStreamIgnored() { // We do not want system data streams coming back in the deprecation info API int oldIndexCount = randomIntBetween(1, 100); @@ -319,7 +488,7 @@ public void testOldSystemDataStreamIgnored() { nameToIndexMetadata.put(oldIndexMetadata.getIndex().getName(), oldIndexMetadata); } for (int i = 0; i < newIndexCount; i++) { - Index newIndex = createNewIndex(i, false, nameToIndexMetadata); + Index newIndex = createNewIndex(i, false, nameToIndexMetadata, MappingMetadata.EMPTY_MAPPINGS); allIndices.add(newIndex); } DataStream dataStream = new DataStream( diff --git a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java index 5ea8a9eea6b8e..fdf4af37fed49 100644 --- a/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java +++ b/x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/IndexDeprecationCheckerTests.java @@ -10,6 +10,7 @@ import com.carrotsearch.randomizedtesting.annotations.Name; import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.TransportVersion; import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.DataStreamMetadata; @@ -191,6 +192,76 @@ public void testMultipleOldIndicesCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(randomBoolean() ? TransportVersion.fromId(0) : TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + + public void testLatestIndicesCheckWithPercolatorFields() { + IndexVersion latestVersion = IndexVersion.current(); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(latestVersion)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.current()) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + assertThat(issuesByIndex.isEmpty(), equalTo(true)); + } + private IndexMetadata indexMetadata(String indexName, IndexVersion indexVersion) { return IndexMetadata.builder(indexName) .settings(settings(indexVersion)) @@ -405,6 +476,46 @@ public void testMultipleOldIndicesIgnoredCheckWithTransforms() { assertEquals(expected, issuesByIndex); } + public void testOldIgnoreIndicesCheckWithPercolatorFields() { + IndexVersion olderVersion = IndexVersion.fromId(9000019); + IndexMetadata indexMetadata = IndexMetadata.builder("test") + .settings(settings(olderVersion).put(MetadataIndexStateService.VERIFIED_READ_ONLY_SETTING.getKey(), true)) + .numberOfShards(1) + .numberOfReplicas(0) + .state(indexMetdataState) + .putMapping(""" + { + "properties": { + "query": { + "type": "percolator" + }, + "field": { + "type": "text" + } + } + } + """) + .transportVersion(TransportVersion.fromId(9000019)) + .build(); + ProjectMetadata project = ProjectMetadata.builder(randomProjectIdOrDefault()).put(indexMetadata, true).build(); + DeprecationIssue expected = new DeprecationIssue( + DeprecationIssue.Level.CRITICAL, + "Field mappings with incompatible percolator type", + "https://www.elastic.co/docs/reference/elasticsearch/mapping-reference/percolator#_reindexing_your_percolator_queries", + "The index was created before 9.latest and contains mappings that must be reindexed due to containing percolator fields. " + + "Field [query] is of type [_doc]", + false, + Map.of("reindex_required", true, "excluded_actions", List.of("readOnly")) + ); + Map> issuesByIndex = checker.check( + project, + new DeprecationInfoAction.Request(TimeValue.THIRTY_SECONDS), + emptyPrecomputedData + ); + List issues = issuesByIndex.get("test"); + assertEquals(singletonList(expected), issues); + } + public void testTranslogRetentionSettings() { Settings.Builder settings = settings(IndexVersion.current()); settings.put(IndexSettings.INDEX_TRANSLOG_RETENTION_AGE_SETTING.getKey(), randomPositiveTimeValue()); diff --git a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java index d237e08b30c19..47215174d0639 100644 --- a/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java +++ b/x-pack/plugin/watcher/src/internalClusterTest/java/org/elasticsearch/xpack/watcher/test/AbstractWatcherIntegrationTestCase.java @@ -275,6 +275,7 @@ public void replaceWatcherIndexWithRandomlyNamedIndex(String originalIndexOrAlia newSettings.remove("index.uuid"); newSettings.remove("index.creation_date"); newSettings.remove("index.version.created"); + newSettings.remove("index.transport_version.created"); assertAcked(indicesAdmin().prepareCreate(to).setMapping(mapping.sourceAsMap()).setSettings(newSettings)); ensureGreen(to);