diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 0985f3490d222..336e650717952 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -73,6 +73,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin { public static final Setting CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope); + private static final int GEOIP_INDEX_MAPPINGS_VERSION = 1; static Set DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"); @@ -225,6 +226,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, GEOIP_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index 212a96ff54ff3..a8e8e7bc935ee 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -209,6 +209,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna { builder.startObject("_meta"); builder.field(VERSION_META_KEY, META_VERSION); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.field(DESCRIPTOR_MANAGED_META_KEY, descriptorManaged); builder.field(DESCRIPTOR_INTERNAL_META_KEY, descriptorInternal); builder.endObject(); diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java index 9847534c1c042..7d0095dd81f63 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -169,6 +169,7 @@ public Collection getSystemIndexDescriptors(Settings sett { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index f55b350520106..d4b242e613c95 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -111,6 +111,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, 1) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index f36c2aae7f8ad..f5a2121b2dde9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.indices.TestSystemIndexDescriptor; import org.elasticsearch.indices.TestSystemIndexDescriptorAllowsTemplates; import org.elasticsearch.indices.TestSystemIndexPlugin; @@ -38,8 +39,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -51,7 +50,6 @@ import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; -import static org.elasticsearch.test.XContentTestUtils.convertToXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -352,7 +350,6 @@ private void assertHasAliases(Set aliasNames, String name, String primar /** * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. - * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters. */ private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) { final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) @@ -366,11 +363,7 @@ private void assertMappingsAndSettings(String expectedMappings, String concreteI ); final Map sourceAsMap = mappings.get(concreteIndex).getSourceAsMap(); - try { - assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false))); final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME)) .actionGet(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java index 9f2a9d3a48162..105e9f5ec91f0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java @@ -68,6 +68,7 @@ public Collection getSystemIndexDescriptors(Settings sett { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index 279f56f33c1a7..486aa8d1f5084 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -17,13 +17,12 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; -import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -31,7 +30,6 @@ import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; -import static org.elasticsearch.test.XContentTestUtils.convertToXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -122,7 +120,6 @@ private void triggerClusterStateUpdates() { /** * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. - * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters. */ private void assertMappingsAndSettings(String expectedMappings) { final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) @@ -136,11 +133,7 @@ private void assertMappingsAndSettings(String expectedMappings) { ); final Map sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap(); - try { - assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false))); final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME)) .actionGet(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index 58830f978ee30..0a090fa889a29 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -97,6 +97,7 @@ public static String getOldMappings() { builder.startObject(); { builder.startObject("_meta"); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); builder.field("version", Version.CURRENT.previousMajor().toString()); builder.endObject(); @@ -123,6 +124,7 @@ public static String getNewMappings() { builder.startObject(); { builder.startObject("_meta"); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.field("version", Version.CURRENT.toString()); builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index fdccfa72d785f..4d37cea8bcf98 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -12,6 +12,8 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService; @@ -130,9 +132,13 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable_meta in the index mappings - * that contains the index's mappings' version. + * that contains the index's mappings' {@link Version}. We need to read and write this field for + * backwards compatibility. */ - private final String versionMetaKey; + private final String mappingsNodeVersionMetaKey; + + /** The version meta key for the integer system index mapping version */ + public static final String VERSION_META_KEY = "managed_index_mappings_version"; /** For internally-managed indices, specifies the origin to use when creating or updating the index */ private final String origin; @@ -140,8 +146,11 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable_meta where a version can be found, which indicates the + * @param mappingsNodeVersionMetaKey a mapping key under _meta where a version can be found, which indicates the * Elasticsearch version when the index was created. * @param origin the client origin to use when creating this index. Internal system indices must not provide an origin, while external * system indices must do so. @@ -209,7 +218,7 @@ protected SystemIndexDescriptor( Settings settings, String aliasName, int indexFormat, - String versionMetaKey, + String mappingsNodeVersionMetaKey, String origin, Version minimumNodeVersion, Type type, @@ -262,18 +271,22 @@ protected SystemIndexDescriptor( Objects.requireNonNull(settings, "Must supply settings for a managed system index"); Strings.requireNonEmpty(mappings, "Must supply mappings for a managed system index"); Strings.requireNonEmpty(primaryIndex, "Must supply primaryIndex for a managed system index"); - Strings.requireNonEmpty(versionMetaKey, "Must supply versionMetaKey for a managed system index"); + Strings.requireNonEmpty(mappingsNodeVersionMetaKey, "Must supply nodeVersionMetaKey for a managed system index"); Strings.requireNonEmpty(origin, "Must supply origin for a managed system index"); if (settings.getAsInt(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 0) != indexFormat) { throw new IllegalArgumentException("Descriptor index format does not match index format in managed settings"); } - this.mappingVersion = extractVersionFromMappings(mappings, versionMetaKey); + this.mappingsNodeVersion = extractNodeVersionFromMappings(mappings, mappingsNodeVersionMetaKey); + this.mappingsVersion = extractVersionFromMappings(mappings); + assert mappingsVersion.version >= 0 : "The mappings version must not be negative"; + } else { assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings"; assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings"; assert Objects.isNull(primaryIndex) : "Unmanaged index descriptors should not have a primary index"; - assert Objects.isNull(versionMetaKey) : "Unmanaged index descriptors should not have a version meta key"; - this.mappingVersion = null; + assert Objects.isNull(mappingsNodeVersionMetaKey) : "Unmanaged index descriptors should not have a version meta key"; + this.mappingsNodeVersion = null; + this.mappingsVersion = null; } Objects.requireNonNull(allowedElasticProductOrigins, "allowedProductOrigins must not be null"); @@ -355,7 +368,7 @@ protected SystemIndexDescriptor( throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true."); } this.indexFormat = indexFormat; - this.versionMetaKey = versionMetaKey; + this.mappingsNodeVersionMetaKey = mappingsNodeVersionMetaKey; this.origin = origin; this.minimumNodeVersion = minimumNodeVersion; this.type = type; @@ -450,9 +463,9 @@ public int getIndexFormat() { return this.indexFormat; } - public String getVersionMetaKey() { + public String getMappingsNodeVersionMetaKey() { assert isAutomaticallyManaged() : "Do not request version meta keys for unmanaged system indices"; - return this.versionMetaKey; + return this.mappingsNodeVersionMetaKey; } public Version getMinimumNodeVersion() { @@ -506,11 +519,23 @@ public boolean allowsTemplates() { return allowsTemplates; } - public Version getMappingVersion() { + /** + * Use of the mappings {@link Version} should be replaced with the value returned from {@link #getMappingsVersion()} + * @return Elasticsearch version associated with this descriptor's mappings. + */ + @Deprecated + public Version getMappingsNodeVersion() { + if (isAutomaticallyManaged() == false) { + throw new IllegalStateException(this + " is not managed so there are no mappings or version"); + } + return mappingsNodeVersion; + } + + public MappingsVersion getMappingsVersion() { if (isAutomaticallyManaged() == false) { throw new IllegalStateException(this + " is not managed so there are no mappings or version"); } - return mappingVersion; + return mappingsVersion; } /** @@ -612,6 +637,20 @@ public boolean isInternal() { } } + /** + * The version of the mapping, which should be stored as an int in a mapping metadata + * field. This will be used with prior index descriptors to determine which mappings + * should be used when creating a system index, and it will be used to determine when + * mappings should be updated with the latest mappings from the system index descriptor. + * See {@link SystemIndexMappingUpdateService}, {@link TransportCreateIndexAction}, + * and {@link AutoCreateAction}. + *

+ * Version should be a non-negative integer for a managed index, or -1 for unmanaged indices. + * The hash is a hash of the system index descriptor's mappings so that we can warn + * in case of inconsistencies across nodes. + */ + public record MappingsVersion(int version, int hash) {}; + /** * Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class. */ @@ -818,8 +857,31 @@ static boolean findDynamicMapping(Map map) { return false; } + @SuppressWarnings("unchecked") // we do a lot of casting of maps + private static MappingsVersion extractVersionFromMappings(String mappings) { + final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, true); + final Map doc = (Map) mappingsMap.get("_doc"); + final Map meta; + final Map properties; + if (doc == null) { + meta = (Map) mappingsMap.get("_meta"); + properties = (Map) mappingsMap.get("properties"); + } else { + meta = (Map) doc.get("_meta"); + properties = (Map) doc.get("properties"); + } + if (meta == null) { + throw new IllegalStateException("mappings do not have _meta field"); + } + final Integer value = (Integer) meta.get(VERSION_META_KEY); + if (value == null) { + throw new IllegalArgumentException("mappings do not have a version in _meta." + VERSION_META_KEY); + } + return new MappingsVersion(value, Objects.hash(properties)); + } + @SuppressWarnings("unchecked") - private static Version extractVersionFromMappings(String mappings, String versionMetaKey) { + private static Version extractNodeVersionFromMappings(String mappings, String versionMetaKey) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false); final Map doc = (Map) mappingsMap.get("_doc"); final Map meta; diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index d5f4d32914179..d1f8acfccc0ac 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -289,7 +289,7 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp return Version.V_EMPTY; } - final Object rawVersion = meta.get(descriptor.getVersionMetaKey()); + final Object rawVersion = meta.get(descriptor.getMappingsNodeVersionMetaKey()); if (rawVersion instanceof Integer) { // This can happen with old system indices, such as .tasks, which were created before we used an Elasticsearch // version here. We should just replace the template to be sure. @@ -297,7 +297,10 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp } final String versionString = rawVersion != null ? rawVersion.toString() : null; if (versionString == null) { - logger.warn("No value found in mappings for [_meta.{}], assuming mappings update required", descriptor.getVersionMetaKey()); + logger.warn( + "No value found in mappings for [_meta.{}], assuming mappings update required", + descriptor.getMappingsNodeVersionMetaKey() + ); // If we called `Version.fromString(null)`, it would return `Version.CURRENT` and we wouldn't update the mappings return Version.V_EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java index b4f84f2fdd1db..848f103aaaf56 100644 --- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java +++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java @@ -81,6 +81,7 @@ public class SynonymsManagementAPIService { public static final int MAX_SYNONYMS_SETS = 10_000; private static final String SYNONYM_RULE_ID_FIELD = SynonymRule.ID_FIELD.getPreferredName(); private static final String SYNONYM_SETS_AGG_NAME = "synonym_sets_aggr"; + private static final int SYNONYMS_INDEX_MAPPINGS_VERSION = 1; private final Client client; @@ -123,6 +124,7 @@ private static XContentBuilder mappings() { builder.startObject("_meta"); { builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SYNONYMS_INDEX_MAPPINGS_VERSION); } builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index d4f4058e358cb..1e42417aa2471 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -66,6 +66,7 @@ public class TaskResultsService { * time is 600000 milliseconds, ten minutes. */ static final BackoffPolicy STORE_BACKOFF_POLICY = BackoffPolicy.exponentialBackoff(timeValueMillis(250), 14); + private static final int TASK_RESULTS_INDEX_MAPPINGS_VERSION = 0; private final Client client; @@ -124,6 +125,7 @@ private static XContentBuilder getTaskResultIndexMappings() { { builder.startObject("_meta"); builder.field(TASK_RESULT_MAPPING_VERSION_META_FIELD, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, TASK_RESULTS_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 2919fd83a446f..6ebdb2729765e 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -274,7 +274,7 @@ public void testNetNewSystemIndicesDontErrorWhenNotRequested() { .setAliasName(".y") .setPrimaryIndex(".b") .setDescription(this.getTestName()) - .setMappings("{\"_meta\": {\"version\": \"1.0.0\"}}") + .setMappings("{\"_meta\": {\"version\": \"1.0.0\", \"" + SystemIndexDescriptor.VERSION_META_KEY + "\": 0}}") .setSettings(Settings.EMPTY) .setVersionMetaKey("version") .setOrigin(this.getTestName()) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 5ccb7900d7ff2..671bf13fb0e77 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -30,6 +30,7 @@ import org.mockito.ArgumentCaptor; import java.util.List; +import java.util.Locale; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; import static org.hamcrest.Matchers.equalTo; @@ -63,7 +64,14 @@ public class TransportCreateIndexActionTests extends ESTestCase { .setPrimaryIndex(MANAGED_SYSTEM_INDEX_NAME + "-primary") .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) .setSettings(SystemIndexDescriptor.DEFAULT_SETTINGS) - .setMappings("{\"_meta\": {\"version\": \"1.0.0\"}}") + .setMappings(String.format(Locale.ROOT, """ + { + "_meta": { + "version": "1.0.0", + "%s": 0 + } + }" + """, SystemIndexDescriptor.VERSION_META_KEY)) .setVersionMetaKey("version") .setOrigin("origin") .build() diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 11b919416c352..64f272c5a071c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -513,7 +514,16 @@ private SystemIndices addSystemIndex(Metadata.Builder mdBuilder) { .setAllowedElasticProductOrigins(List.of("test-net-new-system")) .setNetNew() .setSettings(Settings.EMPTY) - .setMappings("{ \"_doc\": { \"_meta\": { \"version\": \"8.0.0\" } } }") + .setMappings(String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "8.0.0", + "%s": 1 + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY)) .setPrimaryIndex(".test-net-new-system-1") .setVersionMetaKey("version") .setOrigin("system") diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java index 9b4307aae9c3e..bfdef69a5275a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java @@ -19,6 +19,7 @@ import org.junit.Before; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -26,7 +27,16 @@ public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { - private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; + private static final String MAPPINGS = String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "%s": 0 + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY); private static final String SYSTEM_INDEX_NAME = ".myindex-1"; private static final String SYSTEM_ALIAS_NAME = ".myindex-alias"; private static final SystemIndexDescriptor DESCRIPTOR = SystemIndexDescriptor.builder() diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 71239af291ada..7a651dea0c878 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -19,15 +19,26 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.indices.SystemIndexDescriptor.findDynamicMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class SystemIndexDescriptorTests extends ESTestCase { - private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; + private static final String MAPPINGS = String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "%s": 1 + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY); /** * Tests the various validation rules that are applied when creating a new system index descriptor. @@ -243,7 +254,6 @@ public void testPriorSystemIndexDescriptorValidation() { } public void testGetDescriptorCompatibleWith() { - final String mappings = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; final SystemIndexDescriptor prior = SystemIndexDescriptor.builder() .setIndexPattern(".system*") .setDescription("system stuff") @@ -251,7 +261,7 @@ public void testGetDescriptorCompatibleWith() { .setAliasName(".system") .setType(Type.INTERNAL_MANAGED) .setSettings(Settings.EMPTY) - .setMappings(mappings) + .setMappings(MAPPINGS) .setVersionMetaKey("version") .setOrigin("system") .setMinimumNodeVersion(Version.V_7_0_0) @@ -263,7 +273,7 @@ public void testGetDescriptorCompatibleWith() { .setAliasName(".system") .setType(Type.INTERNAL_MANAGED) .setSettings(Settings.EMPTY) - .setMappings(mappings) + .setMappings(MAPPINGS) .setVersionMetaKey("version") .setOrigin("system") .setPriorSystemIndexDescriptors(List.of(prior)) @@ -343,6 +353,79 @@ public void testManagedSystemIndexMustHaveMatchingIndexFormat() { assertThat(e.getMessage(), equalTo("Descriptor index format does not match index format in managed settings")); } + public void testUnmanagedIndexMappingsVersion() { + SystemIndexDescriptor indexDescriptor = SystemIndexDescriptor.builder() + .setIndexPattern(".unmanaged-*") + .setDescription("an unmanaged system index") + .setType(Type.INTERNAL_UNMANAGED) + .build(); + + IllegalStateException e = expectThrows(IllegalStateException.class, indexDescriptor::getMappingsVersion); + + assertThat(e.getMessage(), containsString("is not managed so there are no mappings or version")); + } + + // test mapping versions can't be negative + public void testNegativeMappingsVersion() { + int negativeVersion = randomIntBetween(Integer.MIN_VALUE, -1); + String mappings = String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "%s": %d + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY, negativeVersion); + + SystemIndexDescriptor.Builder builder = priorSystemIndexDescriptorBuilder().setMappings(mappings); + + AssertionError e = expectThrows(AssertionError.class, builder::build); + + assertThat(e.getMessage(), equalTo("The mappings version must not be negative")); + } + + public void testHashesIgnoreMappingMetadata() { + String mappingFormatString = """ + { + "_doc": { + "_meta": { + "version": "%s", + "%s": %d + } + }, + "properties": { + "age": { "type": "integer" }, + "email": { "type": "keyword" }, + "name": { "type": "text" } + } + } + """; + + String mappings1 = String.format( + Locale.ROOT, + mappingFormatString, + "8.9.0", + SystemIndexDescriptor.VERSION_META_KEY, + randomIntBetween(1, 10) + ); + String mappings2 = String.format( + Locale.ROOT, + mappingFormatString, + "8.10.0", + SystemIndexDescriptor.VERSION_META_KEY, + randomIntBetween(11, 20) + ); + + SystemIndexDescriptor descriptor1 = priorSystemIndexDescriptorBuilder().setMappings(mappings1).build(); + SystemIndexDescriptor descriptor2 = priorSystemIndexDescriptorBuilder().setMappings(mappings2).build(); + + assertThat(descriptor1.getMappingsVersion().hash(), equalTo(descriptor2.getMappingsVersion().hash())); + assertThat(descriptor1.getMappingsVersion().version(), not(equalTo(descriptor2.getMappingsVersion().version()))); + assertThat(descriptor1.getMappingsNodeVersion(), not(equalTo(descriptor2.getMappingsNodeVersion()))); + } + private SystemIndexDescriptor.Builder priorSystemIndexDescriptorBuilder() { return SystemIndexDescriptor.builder() .setIndexPattern(".system*") diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java index 330615aae93f5..b3492b203d354 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java @@ -409,7 +409,10 @@ private static XContentBuilder getMappings() { } private static XContentBuilder getMappings(String version) { - return getMappings(builder -> builder.object("_meta", meta -> meta.field("version", version))); + return getMappings(builder -> builder.object("_meta", meta -> { + meta.field("version", version); + meta.field(SystemIndexDescriptor.VERSION_META_KEY, 5); + })); } // Prior to 7.12.0, .tasks had _meta.version: 3 so we need to be sure we can handle that diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index 4e7a2416c4860..be6c1a29bcc6f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -88,6 +88,7 @@ public final class AsyncTaskIndexService> { public static final String RESPONSE_HEADERS_FIELD = "response_headers"; public static final String EXPIRATION_TIME_FIELD = "expiration_time"; public static final String RESULT_FIELD = "result"; + private static final int ASYNC_TASK_INDEX_MAPPINGS_VERSION = 0; // Usually the settings, mappings and system index descriptor below // would be co-located with the SystemIndexPlugin implementation, @@ -110,6 +111,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, ASYNC_TASK_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 2499734672053..da905ef5efdee 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -12,12 +12,15 @@ import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class MlConfigIndex { private static final String INDEX_NAME = ".ml-config"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; + public static final int CONFIG_INDEX_MAPPINGS_VERSION = 1; /** * The name of the index where job, datafeed and analytics configuration is stored @@ -29,7 +32,12 @@ public static String indexName() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/config_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/config_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION)) + ); } public static Settings settings() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index a72a051414677..9190be89ee1e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -11,10 +11,13 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class MlMetaIndex { private static final String INDEX_NAME = ".ml-meta"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int META_INDEX_MAPPINGS_VERSION = 1; /** * Where to store the ml info in Elasticsearch - must match what's @@ -27,7 +30,12 @@ public static String indexName() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/meta_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/meta_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(META_INDEX_MAPPINGS_VERSION)) + ); } public static Settings settings() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java index 76234954b4107..83c7a3fe45a09 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Locale; +import java.util.Map; /** * Describes the indices where ML is storing various stats about the users jobs. @@ -25,6 +26,7 @@ public class MlStatsIndex { public static final String TEMPLATE_NAME = ".ml-stats"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + public static final int STATS_INDEX_MAPPINGS_VERSION = 1; private MlStatsIndex() {} @@ -36,7 +38,12 @@ public static String wrappedMapping() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/stats_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/stats_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(STATS_INDEX_MAPPINGS_VERSION)) + ); } public static String indexPattern() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index c509e52675e38..745e50c920288 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -31,6 +31,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.List; +import java.util.Map; import java.util.SortedMap; import static java.lang.Thread.currentThread; @@ -56,6 +57,7 @@ public class AnnotationIndex { public static final List OLD_INDEX_NAMES = List.of(".ml-annotations-6"); private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + public static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 1; /** * Create the .ml-annotations-6 index with correct mappings if it does not already exist. This index is read and written by the UI @@ -102,7 +104,8 @@ public static void createAnnotationsIndexIfNecessary( client, state, masterNodeTimeout, - finalListener + finalListener, + ANNOTATION_INDEX_MAPPINGS_VERSION ), finalListener::onFailure ); @@ -208,6 +211,11 @@ public static void createAnnotationsIndexIfNecessary( } public static String annotationsMapping() { - return TemplateUtils.loadTemplate("/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/annotations_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION)) + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index aa8fa7c88e719..1e60da895c0e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -12,6 +12,8 @@ import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + /** * Class containing the index constants so that the index version, name, and prefix are available to a wider audience. */ @@ -44,9 +46,15 @@ public final class InferenceIndexConstants { private static final String NATIVE_LATEST_INDEX = NATIVE_INDEX_PREFIX + NATIVE_INDEX_VERSION; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + public static final int INFERENCE_INDEX_MAPPINGS_VERSION = 1; public static String mapping() { - return TemplateUtils.loadTemplate("/ml/inference_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/inference_index_mappings.json", + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION)) + ); } public static String nativeDefinitionStore() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 0dbed16d3b821..29ac3359c870b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Locale; +import java.util.Map; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -29,6 +30,7 @@ public final class AnomalyDetectorsIndex { private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; private static final String RESOURCE_PATH = "/ml/anomalydetection/"; + public static final int RESULTS_INDEX_MAPPINGS_VERSION = 1; private AnomalyDetectorsIndex() {} @@ -136,7 +138,8 @@ public static String resultsMapping() { return TemplateUtils.loadTemplate( RESOURCE_PATH + "results_index_mappings.json", Version.CURRENT.toString(), - RESULTS_MAPPINGS_VERSION_VARIABLE + RESULTS_MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(RESULTS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 1bb88f4bc4773..1dde9dc6075d0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; @@ -22,6 +21,7 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transports; @@ -100,7 +100,7 @@ public class ElasticsearchMappings { private ElasticsearchMappings() {} - static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) { + static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, int minVersion) { List indicesToUpdate = new ArrayList<>(); Map currentMapping = state.metadata() @@ -113,22 +113,26 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic @SuppressWarnings("unchecked") Map meta = (Map) metadata.sourceAsMap().get("_meta"); if (meta != null) { - String versionString = (String) meta.get("version"); - if (versionString == null) { - logger.info("Version of mappings for [{}] not found, recreating", index); + Integer systemIndexMappingsVersion = (Integer) meta.get(SystemIndexDescriptor.VERSION_META_KEY); + if (systemIndexMappingsVersion == null) { + logger.info("System index mappings version for [{}] not found, recreating", index); indicesToUpdate.add(index); continue; } - Version mappingVersion = Version.fromString(versionString); - - if (mappingVersion.onOrAfter(minVersion)) { + if (systemIndexMappingsVersion >= minVersion) { continue; } else { - logger.info("Mappings for [{}] are outdated [{}], updating it[{}].", index, mappingVersion, Version.CURRENT); + logger.info( + "Mappings for [{}] are outdated [{}], updating it[{}].", + index, + systemIndexMappingsVersion, + minVersion + ); indicesToUpdate.add(index); continue; } + } else { logger.info("Version of mappings for [{}] not found, recreating", index); indicesToUpdate.add(index); @@ -153,7 +157,8 @@ public static void addDocMappingIfMissing( Client client, ClusterState state, TimeValue masterNodeTimeout, - ActionListener listener + ActionListener listener, + int minVersion ) { IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(alias); if (indexAbstraction == null) { @@ -167,7 +172,7 @@ public static void addDocMappingIfMissing( protected void doRun() throws Exception { String[] concreteIndices = indexAbstraction.getIndices().stream().map(Index::getName).toArray(String[]::new); - final String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT); + final String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, minVersion); if (indicesThatRequireAnUpdate.length > 0) { String mapping = mappingSupplier.get(); PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 11c4ec2aa0b77..06a923cd9d275 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -9,12 +9,15 @@ import org.elasticsearch.Version; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class NotificationsIndex { public static final String NOTIFICATIONS_INDEX = ".ml-notifications-000002"; private static final String RESOURCE_PATH = "/ml/"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int NOTIFICATIONS_INDEX_MAPPINGS_VERSION = 1; private NotificationsIndex() {} @@ -22,7 +25,8 @@ public static String mapping() { return TemplateUtils.loadTemplate( RESOURCE_PATH + "notifications_index_mappings.json", Version.CURRENT.toString(), - MAPPINGS_VERSION_VARIABLE + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.managed.index.version", Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index cf9708505d075..d95c94801e52a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -23,9 +23,9 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; @@ -130,25 +130,45 @@ public void testMappingRequiresUpdateNoMapping() { ClusterState cs = csBuilder.build(); String[] indices = new String[] { "no_index" }; - assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNullMapping() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("null_mapping", null)); String[] indices = new String[] { "null_index" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNoVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("no_version_field", "NO_VERSION_FIELD")); String[] indices = new String[] { "no_version_field" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateRecentMappingVersion() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_current", Version.CURRENT.toString())); - String[] indices = new String[] { "version_current" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2)); + } } public void testMappingRequiresUpdateMaliciousMappingVersion() { @@ -156,28 +176,53 @@ public void testMappingRequiresUpdateMaliciousMappingVersion() { Collections.singletonMap("version_current", Collections.singletonMap("nested", "1.0")) ); String[] indices = new String[] { "version_nested" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateBogusMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0")); String[] indices = new String[] { "version_bogus" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNewerMappingVersion() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT)); - String[] indices = new String[] { "version_newer" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", 2)); + String[] indices = new String[] { "version_newer" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 2) + ); + String[] indices = new String[] { "version_newer" }; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_newer" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2)); + } } public void testMappingRequiresUpdateNewerMappingVersionMinor() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); - String[] indices = new String[] { "version_newer_minor" }; - assertArrayEquals( - new String[] {}, - ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) - ); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", 1)); + String[] indices = new String[] { "version_newer_minor" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer_minor", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_newer_minor" }; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } } @SuppressWarnings({ "unchecked" }) @@ -200,7 +245,8 @@ public void testAddDocMappingIfMissing() { client, clusterState, MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT, - ActionTestUtils.assertNoFailureListener(Assert::assertTrue) + ActionTestUtils.assertNoFailureListener(Assert::assertTrue), + 1 ); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(PutMappingRequest.class); @@ -215,6 +261,10 @@ public void testAddDocMappingIfMissing() { } private ClusterState getClusterStateWithMappingsWithMetadata(Map namesAndVersions) { + return getClusterStateWithMappingsWithMetadata(namesAndVersions, null); + } + + private ClusterState getClusterStateWithMappingsWithMetadata(Map namesAndVersions, Map metaData) { Metadata.Builder metadataBuilder = Metadata.builder(); for (Map.Entry entry : namesAndVersions.entrySet()) { @@ -236,6 +286,9 @@ private ClusterState getClusterStateWithMappingsWithMetadata(Map if (version != null && version.equals("NO_VERSION_FIELD") == false) { meta.put("version", version); } + if (metaData != null) { + metaData.forEach((k, v) -> meta.putIfAbsent(k, v)); + } mapping.put("_meta", meta); indexMetadata.putMapping(new MappingMetadata("_doc", mapping)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 38bdb8ef18350..92c3a65ffeaa3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -266,6 +266,7 @@ private static XContentBuilder mockMappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, 0) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index 04e67d6ebe528..88c51a9aef284 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -10,7 +10,8 @@ }, "mappings": { "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index 55a6575e15c2a..2b3ecbac92352 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 0ba3f384ad6bc..6b96b204b4f7f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index bb7fbda297db2..4a5f76470d24f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 2032bee4aaabb..6be455e02825a 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index 73179d529aae8..7cce4e0d4f53a 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index 3e7031c1eab89..44e2e67dd06c3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 684dfc1c7928a..6d16098e38049 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index b9b9a8f5b5249..8ddd5e7e0693e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "agent": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index 1292ce06e3044..42efa741dd99d 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta" : { - "version" : "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json index 89389012664b2..4415afe50a998 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json @@ -1,6 +1,7 @@ { "_meta" : { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index 7de21b4901f1a..811810f5d0f2f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -1,7 +1,8 @@ { "_doc" : { "_meta" : { - "version" : "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index d2e73d494bcc3..7ff961a0aac9c 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index f78414f53ea4f..4606cb0d75d8f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index e0e44d8b230f4..b2135d2c6e76e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,6 +1,7 @@ { "_meta" : { - "version" : "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index 711bfe1f9c1dc..8c151cb38c1b3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,6 +1,7 @@ { "_meta": { - "version" : "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic": false, "properties" : { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index d629700b7c61a..de823ab2a6ce2 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -63,6 +63,7 @@ public class QueryRulesIndexService { public static final String QUERY_RULES_ALIAS_NAME = ".query-rules"; public static final String QUERY_RULES_CONCRETE_INDEX_NAME = ".query-rules-1"; public static final String QUERY_RULES_INDEX_NAME_PATTERN = ".query-rules-*"; + private static final int QUERY_RULES_INDEX_MAPPINGS_VERSION = 1; private final Client clientWithOrigin; private final ClusterSettings clusterSettings; @@ -107,6 +108,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, QUERY_RULES_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java index b4917607feb88..681a2ff5028a3 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java @@ -94,6 +94,7 @@ public class SearchApplicationIndexService { public static final String SEARCH_APPLICATION_ALIAS_NAME = ".search-app"; public static final String SEARCH_APPLICATION_CONCRETE_INDEX_NAME = ".search-app-1"; public static final String SEARCH_APPLICATION_INDEX_NAME_PATTERN = ".search-app-*"; + private static final int SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION = 1; // The client to perform any operations on user indices (alias, ...). private final Client client; @@ -152,6 +153,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 4afc2bb44793a..009e17bd79e6b 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -89,6 +89,15 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final String VERSION_KEY = "version"; private static final String MAPPING_VERSION_VARIABLE = "fleet.version"; private static final List ALLOWED_PRODUCTS = List.of("kibana", "fleet"); + private static final int FLEET_ACTIONS_MAPPINGS_VERSION = 1; + private static final int FLEET_AGENTS_MAPPINGS_VERSION = 1; + private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 1; + private static final int FLEET_SECRETS_MAPPINGS_VERSION = 1; + private static final int FLEET_POLICIES_MAPPINGS_VERSION = 1; + private static final int FLEET_POLICIES_LEADER_MAPPINGS_VERSION = 1; + private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1; + private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1; + private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1; @Override public Collection createComponents( @@ -149,7 +158,7 @@ public String getFeatureDescription() { private SystemIndexDescriptor fleetActionsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-actions.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-actions.json", FLEET_ACTIONS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -167,7 +176,7 @@ private SystemIndexDescriptor fleetActionsSystemIndexDescriptor() { private SystemIndexDescriptor fleetAgentsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-agents.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-agents.json", FLEET_AGENTS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -185,7 +194,10 @@ private SystemIndexDescriptor fleetAgentsSystemIndexDescriptor() { private SystemIndexDescriptor fleetEnrollmentApiKeysSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-enrollment-api-keys.json"), XContentType.JSON); + request.source( + loadTemplateSource("/fleet-enrollment-api-keys.json", FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION), + XContentType.JSON + ); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -203,7 +215,7 @@ private SystemIndexDescriptor fleetEnrollmentApiKeysSystemIndexDescriptor() { private SystemIndexDescriptor fleetSecretsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-secrets.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-secrets.json", FLEET_SECRETS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.INTERNAL_MANAGED) .setOrigin(FLEET_ORIGIN) @@ -219,7 +231,7 @@ private SystemIndexDescriptor fleetSecretsSystemIndexDescriptor() { private SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-policies.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-policies.json", FLEET_POLICIES_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -237,7 +249,7 @@ private SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() { private SystemIndexDescriptor fleetPoliciesLeaderSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-policies-leader.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-policies-leader.json", FLEET_POLICIES_LEADER_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -255,7 +267,7 @@ private SystemIndexDescriptor fleetPoliciesLeaderSystemIndexDescriptor() { private SystemIndexDescriptor fleetServersSystemIndexDescriptors() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-servers.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-servers.json", FLEET_SERVERS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -273,7 +285,7 @@ private SystemIndexDescriptor fleetServersSystemIndexDescriptors() { private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-artifacts.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-artifacts.json", FLEET_ARTIFACTS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -290,7 +302,7 @@ private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { } private SystemDataStreamDescriptor fleetActionsResultsDescriptor() { - final String source = loadTemplateSource("/fleet-actions-results.json"); + final String source = loadTemplateSource("/fleet-actions-results.json", FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, source)) { ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.parse(parser); return new SystemDataStreamDescriptor( @@ -345,8 +357,13 @@ public void cleanUpFeature(ClusterService clusterService, Client client, ActionL } } - private String loadTemplateSource(String resource) { - return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), MAPPING_VERSION_VARIABLE); + private String loadTemplateSource(String resource, int mappingsVersion) { + return TemplateUtils.loadTemplate( + resource, + Version.CURRENT.toString(), + MAPPING_VERSION_VARIABLE, + Map.of("fleet.managed.index.version", Integer.toString(mappingsVersion)) + ); } @Override diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 4eb9d058aa2dc..ecba96432839b 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -58,6 +58,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin { public static final String LOGSTASH_CONCRETE_INDEX_NAME = ".logstash"; public static final String LOGSTASH_INDEX_NAME_PATTERN = LOGSTASH_CONCRETE_INDEX_NAME + "*"; + public static final int LOGSTASH_INDEX_MAPPINGS_VERSION = 1; static final LicensedFeature.Momentary LOGSTASH_FEATURE = LicensedFeature.momentary(null, "logstash", License.OperationMode.STANDARD); @@ -121,6 +122,7 @@ private XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("logstash-version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, LOGSTASH_INDEX_MAPPINGS_VERSION); builder.endObject(); } { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index a458c3df86910..5e033278242ad 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -313,7 +313,8 @@ private void clearJobFinishedTime( client, clusterState, masterNodeTimeout, - mappingsUpdatedListener + mappingsUpdatedListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index bff2d7dc69439..9ae73224a698d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -273,7 +273,8 @@ private void updateDocMappingAndPutConfig( client, clusterState, masterNodeTimeout, - ActionListener.wrap(unused -> configProvider.put(config, headers, masterNodeTimeout, auditingListener), listener::onFailure) + ActionListener.wrap(unused -> configProvider.put(config, headers, masterNodeTimeout, auditingListener), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index ecfbec71ab285..b12d1ffd75287 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -169,7 +169,8 @@ protected void masterOperation( client, state, request.masterNodeTimeout(), - configMappingUpdateListener + configMappingUpdateListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java index c542213029289..921adf3924dde 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java @@ -110,7 +110,8 @@ protected void masterOperation( client, state, request.masterNodeTimeout(), - ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java index a97ce5628da23..bd6b55d10af5a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java @@ -174,7 +174,8 @@ protected void masterOperation(Task task, Request request, ClusterState state, A client, state, request.masterNodeTimeout(), - configIndexMappingUpdaterListener + configIndexMappingUpdaterListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java index 7fcf7cf980246..ef8b589735585 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java @@ -239,7 +239,8 @@ public void updateDatafeed( client, state, request.masterNodeTimeout(), - ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } @@ -332,7 +333,8 @@ private void putDatafeed( client, clusterState, request.masterNodeTimeout(), - ActionListener.wrap(mappingsUpdated, listener::onFailure) + ActionListener.wrap(mappingsUpdated, listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); }; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index e699f071e2cfc..718b70fb8801b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -165,7 +165,8 @@ private void createStatsIndexAndUpdateMappingsIfNecessary( clientToUse, clusterState, masterNodeTimeout, - listener + listener, + MlStatsIndex.STATS_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java index 6376b21f07c06..b6c6f0865aad8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java @@ -262,7 +262,8 @@ private void createStatsIndexIfNecessary() { client, clusterState, MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT, - listener + listener, + MlStatsIndex.STATS_INDEX_MAPPINGS_VERSION ), listener::onFailure ) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index c3ffd841eadf3..ba3a52df5cdb6 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -282,7 +282,8 @@ public void onFailure(Exception e) { client, state, request.masterNodeTimeout(), - putJobListener + putJobListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), putJobListener::onFailure ); @@ -353,7 +354,8 @@ public void updateJob(UpdateJobAction.Request request, ActionListener doUpdate.run(), actionListener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), actionListener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); if (request.getJobUpdate().getGroups() != null && request.getJobUpdate().getGroups().isEmpty() == false) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 9170950778444..f09b3166964f5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -574,7 +574,8 @@ public void openJob( client, clusterState, masterNodeTimeout, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ), e -> { // Due to a bug in 7.9.0 it's possible that the annotations index already has incorrect mappings @@ -586,7 +587,8 @@ public void openJob( client, clusterState, masterNodeTimeout, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java index 8119deb7674f9..69b926876302a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java @@ -198,7 +198,8 @@ protected void nodeOperation(AllocatedPersistentTask task, SnapshotUpgradeTaskPa client, clusterState, MlTasks.PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ), e -> { // Due to a bug in 7.9.0 it's possible that the annotations index already has incorrect mappings @@ -210,7 +211,8 @@ protected void nodeOperation(AllocatedPersistentTask task, SnapshotUpgradeTaskPa client, clusterState, MlTasks.PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java index 8b990f22c7b38..7a027e2c5cfa4 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java @@ -277,7 +277,8 @@ protected void nodeOperation(AllocatedPersistentTask task, OpenJobAction.JobPara client, clusterState, PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 9866f5e3e2a84..35c7e703bf008 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -251,6 +251,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng * This affects the system searchable snapshot cache index (not the searchable snapshot index itself) */ public static final String DATA_TIERS_CACHE_INDEX_PREFERENCE = String.join(",", DataTier.DATA_CONTENT, DataTier.DATA_HOT); + private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION = 1; private volatile Supplier repositoriesServiceSupplier; private final SetOnce blobStoreCacheService = new SetOnce<>(); @@ -613,6 +614,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION); builder.endObject(); } { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 1277d99c2acca..bdc25098f1760 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -264,7 +264,7 @@ private boolean checkIndexMappingUpToDate(ClusterState clusterState) { * * ...which is true if the mappings have been updated. */ - return checkIndexMappingVersionMatches(clusterState, descriptor.getMappingVersion()::onOrBefore); + return checkIndexMappingVersionMatches(clusterState, descriptor.getMappingsNodeVersion()::onOrBefore); } private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate predicate) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index c3128fff5505e..594726b47a85e 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -37,8 +37,11 @@ public class SecuritySystemIndices { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; + private static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 1; private static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; + private static final int INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT = 1; private static final int INTERNAL_PROFILE_INDEX_FORMAT = 8; + private static final int INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT = 1; public static final String SECURITY_MAIN_ALIAS = ".security"; private static final String MAIN_INDEX_CONCRETE_NAME = ".security-7"; @@ -142,6 +145,7 @@ private XContentBuilder getMainIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); @@ -627,6 +631,7 @@ private static XContentBuilder getTokenIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); @@ -838,6 +843,7 @@ private XContentBuilder getProfileIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 783aadedcd699..722f37059cc56 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -87,6 +87,7 @@ public final class TransformInternalIndex { public static final String KEYWORD = "keyword"; public static final String BOOLEAN = "boolean"; public static final String FLATTENED = "flattened"; + public static final int TRANSFORM_INDEX_MAPPINGS_VERSION = 1; public static SystemIndexDescriptor getSystemIndexDescriptor(Settings transformInternalIndexAdditionalSettings) throws IOException { return SystemIndexDescriptor.builder() @@ -357,7 +358,10 @@ private static XContentBuilder addTransformCheckpointMappings(XContentBuilder bu * @throws IOException On write error */ private static XContentBuilder addMetaInformation(XContentBuilder builder) throws IOException { - return builder.startObject("_meta").field("version", Version.CURRENT).endObject(); + return builder.startObject("_meta") + .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, TRANSFORM_INDEX_MAPPINGS_VERSION) + .endObject(); } protected static boolean hasLatestVersionedIndex(ClusterState state) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index ca3a68f99d74d..afcc8a2097e94 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -276,6 +276,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, ); private static final Logger logger = LogManager.getLogger(Watcher.class); + private static final int WATCHES_INDEX_MAPPINGS_VERSION = 1; + private static final int TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION = 1; private WatcherIndexingListener listener; private HttpClient httpClient; private BulkProcessor2 bulkProcessor; @@ -876,6 +878,7 @@ private XContentBuilder getWatchesIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } { @@ -967,6 +970,7 @@ private XContentBuilder getTriggeredWatchesIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 39efc221ed129..95d543a0f63ea 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -10,6 +10,9 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; @@ -129,6 +132,10 @@ private void assertUpgradedResultsMappings() throws Exception { assertNotNull(indexLevel); assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals( + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword" @@ -162,6 +169,10 @@ private void assertUpgradedAnnotationsMappings() throws Exception { assertNotNull(indexLevel); assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals( + AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that would be incorrectly mapped by dynamic @@ -216,6 +227,10 @@ private void assertUpgradedConfigMappings() throws Exception { assertNotNull(indexLevel); assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals( + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword"