From 420b6500cb3a4cd1993b4f85be397a7910f39ff2 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 19 Jul 2023 17:31:37 -0400 Subject: [PATCH 01/34] TODOS --- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 7 +++++++ .../indices/SystemIndexMappingUpdateService.java | 3 +++ 2 files changed, 10 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index fdccfa72d785f..a3fd70aa52ce3 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -132,15 +132,18 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable_meta in the index mappings * that contains the index's mappings' version. */ + // TODO[wrb]: need to keep this for bwc & finding the field to update, but we can standardize a key under _meta private final String versionMetaKey; /** For internally-managed indices, specifies the origin to use when creating or updating the index */ private final String origin; /** The minimum cluster node version required for this descriptor */ + // TODO[wrb]: we can remove this once the mapping version is used for it private final Version minimumNodeVersion; /** Mapping version from the descriptor */ + // TODO[wrb]: Needs to be a new class, int-version/hash private final Version mappingVersion; /** Whether there are dynamic fields in this descriptor's mappings */ @@ -267,6 +270,7 @@ protected SystemIndexDescriptor( if (settings.getAsInt(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 0) != indexFormat) { throw new IllegalArgumentException("Descriptor index format does not match index format in managed settings"); } + // TODO[wrb]: extract version int and non-metadata hash something different this.mappingVersion = extractVersionFromMappings(mappings, versionMetaKey); } else { assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings"; @@ -295,6 +299,7 @@ protected SystemIndexDescriptor( Set versions = Sets.newHashSetWithExpectedSize(priorSystemIndexDescriptors.size() + 1); versions.add(minimumNodeVersion); for (SystemIndexDescriptor prior : priorSystemIndexDescriptors) { + // TODO[wrb]: use mapping version instead of minimum node version if (versions.add(prior.minimumNodeVersion) == false) { throw new IllegalArgumentException(prior + " has the same minimum node version as another descriptor"); } @@ -506,6 +511,7 @@ public boolean allowsTemplates() { return allowsTemplates; } + // TODO[wrb]: return something different public Version getMappingVersion() { if (isAutomaticallyManaged() == false) { throw new IllegalStateException(this + " is not managed so there are no mappings or version"); @@ -520,6 +526,7 @@ public Version getMappingVersion() { * @param cause the action being attempted that triggered the check. Used in the error message. * @return the standardized error message */ + // TODO[wrb]: update for new "mapping version" world public String getMinimumNodeVersionMessage(String cause) { Objects.requireNonNull(cause); final Version actualMinimumVersion = priorSystemIndexDescriptors.isEmpty() diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index d5f4d32914179..9dd2ba4b43275 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -93,6 +93,7 @@ public void clusterChanged(ClusterChangedEvent event) { } // if we're in a mixed-version cluster, exit + // TODO[wrb]: compare cluster minimum mapping version for system index descriptors if (state.nodes().getMaxNodeVersion().after(state.nodes().getSmallestNonClientNodeVersion())) { logger.debug("Skipping system indices up-to-date check as cluster has mixed versions"); return; @@ -267,12 +268,14 @@ private static boolean checkIndexMappingUpToDate(SystemIndexDescriptor descripto return false; } + // TODO[wrb]: compare _descriptor_ mapping version with mapping in cluster return Version.CURRENT.onOrBefore(readMappingVersion(descriptor, mappingMetadata)); } /** * Fetches the mapping version from an index's mapping's `_meta` info. */ + // TODO[wrb]: read the new kind of mapping version private static Version readMappingVersion(SystemIndexDescriptor descriptor, MappingMetadata mappingMetadata) { final String indexName = descriptor.getPrimaryIndex(); try { From 441a2e9c73dd0f4e42e97e0addb184076167f987 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Mon, 24 Jul 2023 15:47:54 -0400 Subject: [PATCH 02/34] Add modules version so tests will pass --- .../ingest/geoip/IngestGeoIpPlugin.java | 2 + .../indices/SystemIndexDescriptor.java | 68 ++++++++++++++++--- .../SystemIndexMappingUpdateService.java | 7 +- .../SynonymsManagementAPIService.java | 1 + .../tasks/TaskResultsService.java | 1 + .../indices/SystemIndexDescriptorTests.java | 17 +++-- .../SystemIndexMappingUpdateServiceTests.java | 5 +- .../core/async/AsyncTaskIndexService.java | 2 + .../main/resources/fleet-actions-results.json | 3 +- .../src/main/resources/fleet-actions.json | 3 +- .../src/main/resources/fleet-agents.json | 3 +- .../src/main/resources/fleet-artifacts.json | 3 +- .../resources/fleet-enrollment-api-keys.json | 3 +- .../main/resources/fleet-policies-leader.json | 3 +- .../src/main/resources/fleet-policies.json | 3 +- .../src/main/resources/fleet-secrets.json | 3 +- .../src/main/resources/fleet-servers.json | 3 +- .../ml/annotations_index_mappings.json | 3 +- .../resources/ml/config_index_mappings.json | 3 +- .../ml/inference_index_mappings.json | 3 +- .../resources/ml/meta_index_mappings.json | 3 +- .../ml/notifications_index_mappings.json | 3 +- .../resources/ml/stats_index_mappings.json | 3 +- .../rules/QueryRulesIndexService.java | 2 + .../search/SearchApplicationIndexService.java | 2 + .../xpack/logstash/Logstash.java | 2 + .../SearchableSnapshots.java | 2 + .../support/SecurityIndexManager.java | 2 +- .../support/SecuritySystemIndices.java | 6 ++ .../persistence/TransformInternalIndex.java | 6 +- .../elasticsearch/xpack/watcher/Watcher.java | 4 ++ 31 files changed, 140 insertions(+), 34 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 0985f3490d222..c15a8dd1fd4a7 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -73,6 +73,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin { public static final Setting CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope); + private static final int GEOIP_INDEX_MAPPINGS_VERSION = 0; static Set DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"); @@ -225,6 +226,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, GEOIP_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index a3fd70aa52ce3..43c1d55ca2b0c 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -12,6 +12,8 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.Version; +import org.elasticsearch.action.admin.indices.create.AutoCreateAction; +import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService; @@ -133,7 +135,10 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable + * Version should be a non-negative integer for a managed index, or -1 for unmanaged indices. + * The hash is a hash of the system index descriptor's mappings so that we can warn + * in case of inconsistencies across nodes. + */ + public record MappingVersion(int version, int hash) {}; + /** * Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class. */ @@ -825,8 +849,31 @@ static boolean findDynamicMapping(Map map) { return false; } + @SuppressWarnings("unchecked") // we do a lot of casting of maps + private static MappingVersion extractVersionFromMappings(String mappings) { + final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, true); + final Map doc = (Map) mappingsMap.get("_doc"); + final Map meta; + final Map properties; + if (doc == null) { + meta = (Map) mappingsMap.get("_meta"); + properties = (Map) mappingsMap.get("properties"); + } else { + meta = (Map) doc.get("_meta"); + properties = (Map) doc.get("properties"); + } + if (meta == null) { + throw new IllegalStateException("mappings do not have _meta field"); + } + final Integer value = (Integer) meta.get(VERSION_META_KEY); + if (value == null) { + throw new IllegalArgumentException("mappings do not have a version in _meta." + VERSION_META_KEY); + } + return new MappingVersion(value, Objects.hash(properties)); + } + @SuppressWarnings("unchecked") - private static Version extractVersionFromMappings(String mappings, String versionMetaKey) { + private static Version extractLegacyVersionFromMappings(String mappings, String versionMetaKey) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false); final Map doc = (Map) mappingsMap.get("_doc"); final Map meta; @@ -840,6 +887,7 @@ private static Version extractVersionFromMappings(String mappings, String versio } final String value = (String) meta.get(versionMetaKey); if (value == null) { + // TODO[wrb]: Tolerate null throw new IllegalArgumentException("mappings do not have a version in _meta." + versionMetaKey); } return Version.fromString(value); diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index 9dd2ba4b43275..b078d308db344 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -292,7 +292,7 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp return Version.V_EMPTY; } - final Object rawVersion = meta.get(descriptor.getVersionMetaKey()); + final Object rawVersion = meta.get(descriptor.getLegacyVersionMetaKey()); if (rawVersion instanceof Integer) { // This can happen with old system indices, such as .tasks, which were created before we used an Elasticsearch // version here. We should just replace the template to be sure. @@ -300,7 +300,10 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp } final String versionString = rawVersion != null ? rawVersion.toString() : null; if (versionString == null) { - logger.warn("No value found in mappings for [_meta.{}], assuming mappings update required", descriptor.getVersionMetaKey()); + logger.warn( + "No value found in mappings for [_meta.{}], assuming mappings update required", + descriptor.getLegacyVersionMetaKey() + ); // If we called `Version.fromString(null)`, it would return `Version.CURRENT` and we wouldn't update the mappings return Version.V_EMPTY; } diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java index 71b324a324bd4..ca8ad2ad4375a 100644 --- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java +++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java @@ -123,6 +123,7 @@ private static XContentBuilder mappings() { builder.startObject("_meta"); { builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); // TODO[wrb]: constant } builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index d4f4058e358cb..06857c894220f 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -124,6 +124,7 @@ private static XContentBuilder getTaskResultIndexMappings() { { builder.startObject("_meta"); builder.field(TASK_RESULT_MAPPING_VERSION_META_FIELD, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); // TODO[wrb]: constant builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 71239af291ada..79b214e9d14f6 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -27,7 +27,17 @@ public class SystemIndexDescriptorTests extends ESTestCase { - private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; + // private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; + private static final String MAPPINGS = """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "system-index-mapping-version": 1 + } + } + } + """; /** * Tests the various validation rules that are applied when creating a new system index descriptor. @@ -243,7 +253,6 @@ public void testPriorSystemIndexDescriptorValidation() { } public void testGetDescriptorCompatibleWith() { - final String mappings = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; final SystemIndexDescriptor prior = SystemIndexDescriptor.builder() .setIndexPattern(".system*") .setDescription("system stuff") @@ -251,7 +260,7 @@ public void testGetDescriptorCompatibleWith() { .setAliasName(".system") .setType(Type.INTERNAL_MANAGED) .setSettings(Settings.EMPTY) - .setMappings(mappings) + .setMappings(MAPPINGS) .setVersionMetaKey("version") .setOrigin("system") .setMinimumNodeVersion(Version.V_7_0_0) @@ -263,7 +272,7 @@ public void testGetDescriptorCompatibleWith() { .setAliasName(".system") .setType(Type.INTERNAL_MANAGED) .setSettings(Settings.EMPTY) - .setMappings(mappings) + .setMappings(MAPPINGS) .setVersionMetaKey("version") .setOrigin("system") .setPriorSystemIndexDescriptors(List.of(prior)) diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java index 8ec01473abb9c..f423f4611f470 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceTests.java @@ -408,7 +408,10 @@ private static XContentBuilder getMappings() { } private static XContentBuilder getMappings(String version) { - return getMappings(builder -> builder.object("_meta", meta -> meta.field("version", version))); + return getMappings(builder -> builder.object("_meta", meta -> { + meta.field("version", version); + meta.field(SystemIndexDescriptor.VERSION_META_KEY, 5); + })); } // Prior to 7.12.0, .tasks had _meta.version: 3 so we need to be sure we can handle that diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index 4e7a2416c4860..be6c1a29bcc6f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -88,6 +88,7 @@ public final class AsyncTaskIndexService> { public static final String RESPONSE_HEADERS_FIELD = "response_headers"; public static final String EXPIRATION_TIME_FIELD = "expiration_time"; public static final String RESULT_FIELD = "result"; + private static final int ASYNC_TASK_INDEX_MAPPINGS_VERSION = 0; // Usually the settings, mappings and system index descriptor below // would be co-located with the SystemIndexPlugin implementation, @@ -110,6 +111,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, ASYNC_TASK_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") .startObject("properties") diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index 04e67d6ebe528..7db3ae387ceea 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -10,7 +10,8 @@ }, "mappings": { "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index 55a6575e15c2a..8c4408633f3f1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 0ba3f384ad6bc..a94400af66324 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index bb7fbda297db2..d52f9b3a37f68 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 2032bee4aaabb..cde4c6c3d020e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index 73179d529aae8..289e8cba588a0 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index 3e7031c1eab89..977f1cd7c1d4c 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 684dfc1c7928a..677d3c05ef559 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index b9b9a8f5b5249..60d4e47115b83 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -6,7 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}" + "version": "${fleet.version}", + "system-index-mapping-version": 0 }, "properties": { "agent": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index 1292ce06e3044..16364adf333f6 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta" : { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index 7de21b4901f1a..bdfbbe880fb50 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -1,7 +1,8 @@ { "_doc" : { "_meta" : { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index d2e73d494bcc3..e7a3d8f1aeca6 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index f78414f53ea4f..321da59bb6fa7 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -1,7 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}" + "version": "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index e0e44d8b230f4..4dbefbbb7a4a2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,6 +1,7 @@ { "_meta" : { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index 711bfe1f9c1dc..fe3df5ed0b843 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,6 +1,7 @@ { "_meta": { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "system-index-mapping-version": 0 }, "dynamic": false, "properties" : { diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index 786c812d1d56a..d3a091e7a2fac 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -62,6 +62,7 @@ public class QueryRulesIndexService { public static final String QUERY_RULES_ALIAS_NAME = ".query-rules"; public static final String QUERY_RULES_CONCRETE_INDEX_NAME = ".query-rules-1"; public static final String QUERY_RULES_INDEX_NAME_PATTERN = ".query-rules-*"; + private static final int QUERY_RULES_INDEX_MAPPINGS_VERSION = 0; private final Client clientWithOrigin; private final ClusterSettings clusterSettings; @@ -106,6 +107,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, QUERY_RULES_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java index 01a95f9b00376..165d2309f8ae0 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java @@ -95,6 +95,7 @@ public class SearchApplicationIndexService { public static final String SEARCH_APPLICATION_ALIAS_NAME = ".search-app"; public static final String SEARCH_APPLICATION_CONCRETE_INDEX_NAME = ".search-app-1"; public static final String SEARCH_APPLICATION_INDEX_NAME_PATTERN = ".search-app-*"; + private static final int SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION = 0; private static final String INCONSISTENT_INDICES_NOT_IN_ALIAS_MSG = "index is in search application but not in associated alias"; private static final String INCONSISTENT_ALIAS_NOT_IN_INDICES_MSG = "index is in alias but not associcated with search application"; @@ -155,6 +156,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 4eb9d058aa2dc..4fe588e039e26 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -58,6 +58,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin { public static final String LOGSTASH_CONCRETE_INDEX_NAME = ".logstash"; public static final String LOGSTASH_INDEX_NAME_PATTERN = LOGSTASH_CONCRETE_INDEX_NAME + "*"; + public static final int LOGSTASH_INDEX_MAPPINGS_VERSION = 0; static final LicensedFeature.Momentary LOGSTASH_FEATURE = LicensedFeature.momentary(null, "logstash", License.OperationMode.STANDARD); @@ -121,6 +122,7 @@ private XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("logstash-version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, LOGSTASH_INDEX_MAPPINGS_VERSION); builder.endObject(); } { diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 9866f5e3e2a84..fa27602bbd9fa 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -251,6 +251,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng * This affects the system searchable snapshot cache index (not the searchable snapshot index itself) */ public static final String DATA_TIERS_CACHE_INDEX_PREFERENCE = String.join(",", DataTier.DATA_CONTENT, DataTier.DATA_HOT); + private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPING_VERSION = 0; private volatile Supplier repositoriesServiceSupplier; private final SetOnce blobStoreCacheService = new SetOnce<>(); @@ -613,6 +614,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCHABLE_SNAPSHOTS_INDEX_MAPPING_VERSION); builder.endObject(); } { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index 1277d99c2acca..edae07a94a902 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -264,7 +264,7 @@ private boolean checkIndexMappingUpToDate(ClusterState clusterState) { * * ...which is true if the mappings have been updated. */ - return checkIndexMappingVersionMatches(clusterState, descriptor.getMappingVersion()::onOrBefore); + return checkIndexMappingVersionMatches(clusterState, descriptor.getOldMappingVersion()::onOrBefore); } private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate predicate) { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index 3926c96193269..e52770275f822 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -36,8 +36,11 @@ public class SecuritySystemIndices { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; + private static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 0; private static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; + private static final int INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT = 0; private static final int INTERNAL_PROFILE_INDEX_FORMAT = 8; + private static final int INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT = 0; public static final String SECURITY_MAIN_ALIAS = ".security"; private static final String MAIN_INDEX_CONCRETE_NAME = ".security-7"; @@ -141,6 +144,7 @@ private XContentBuilder getMainIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); @@ -628,6 +632,7 @@ private static XContentBuilder getTokenIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); @@ -828,6 +833,7 @@ private XContentBuilder getProfileIndexMappings() { { builder.startObject("_meta"); builder.field(SECURITY_VERSION_STRING, Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 783aadedcd699..14ca822d5a493 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -87,6 +87,7 @@ public final class TransformInternalIndex { public static final String KEYWORD = "keyword"; public static final String BOOLEAN = "boolean"; public static final String FLATTENED = "flattened"; + public static final int TRANSFORM_INDEX_MAPPINGS_VERSION = 0; public static SystemIndexDescriptor getSystemIndexDescriptor(Settings transformInternalIndexAdditionalSettings) throws IOException { return SystemIndexDescriptor.builder() @@ -357,7 +358,10 @@ private static XContentBuilder addTransformCheckpointMappings(XContentBuilder bu * @throws IOException On write error */ private static XContentBuilder addMetaInformation(XContentBuilder builder) throws IOException { - return builder.startObject("_meta").field("version", Version.CURRENT).endObject(); + return builder.startObject("_meta") + .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, TRANSFORM_INDEX_MAPPINGS_VERSION) + .endObject(); } protected static boolean hasLatestVersionedIndex(ClusterState state) { diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index ca3a68f99d74d..957adf2bd40f2 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -276,6 +276,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, ); private static final Logger logger = LogManager.getLogger(Watcher.class); + private static final int WATCHES_INDEX_MAPPINGS_VERSION = 0; + private static final int TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION = 0; private WatcherIndexingListener listener; private HttpClient httpClient; private BulkProcessor2 bulkProcessor; @@ -876,6 +878,7 @@ private XContentBuilder getWatchesIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } { @@ -967,6 +970,7 @@ private XContentBuilder getTriggeredWatchesIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } { From 633e58dce09b4913920b59056c0cbef310b0e768 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 10:33:29 -0400 Subject: [PATCH 03/34] Add constant for old-style mapping version --- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 43c1d55ca2b0c..7719797025e36 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -140,6 +140,10 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable= 0; } else { assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings"; assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings"; @@ -521,7 +526,6 @@ public boolean allowsTemplates() { return allowsTemplates; } - // TODO[wrb]: return something different public Version getOldMappingVersion() { if (isAutomaticallyManaged() == false) { throw new IllegalStateException(this + " is not managed so there are no mappings or version"); @@ -872,6 +876,7 @@ private static MappingVersion extractVersionFromMappings(String mappings) { return new MappingVersion(value, Objects.hash(properties)); } + // TODO[wrb]: remove method @SuppressWarnings("unchecked") private static Version extractLegacyVersionFromMappings(String mappings, String versionMetaKey) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false); From 5fc2d00279746c7d939d3bb79a63b689f614fa13 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 10:57:11 -0400 Subject: [PATCH 04/34] Introduce constants --- .../elasticsearch/synonyms/SynonymsManagementAPIService.java | 3 ++- .../main/java/org/elasticsearch/tasks/TaskResultsService.java | 3 ++- .../xpack/searchablesnapshots/SearchableSnapshots.java | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java index ca8ad2ad4375a..5ea60dceeb65e 100644 --- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java +++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java @@ -81,6 +81,7 @@ public class SynonymsManagementAPIService { public static final int MAX_SYNONYMS_SETS = 10_000; private static final String SYNONYM_RULE_ID_FIELD = SynonymRule.ID_FIELD.getPreferredName(); private static final String SYNONYM_SETS_AGG_NAME = "synonym_sets_aggr"; + private static final int SYNONYMS_INDEX_MAPPINGS_VERSION = 0; private final Client client; @@ -123,7 +124,7 @@ private static XContentBuilder mappings() { builder.startObject("_meta"); { builder.field("version", Version.CURRENT.toString()); - builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); // TODO[wrb]: constant + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SYNONYMS_INDEX_MAPPINGS_VERSION); } builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index 06857c894220f..1e42417aa2471 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -66,6 +66,7 @@ public class TaskResultsService { * time is 600000 milliseconds, ten minutes. */ static final BackoffPolicy STORE_BACKOFF_POLICY = BackoffPolicy.exponentialBackoff(timeValueMillis(250), 14); + private static final int TASK_RESULTS_INDEX_MAPPINGS_VERSION = 0; private final Client client; @@ -124,7 +125,7 @@ private static XContentBuilder getTaskResultIndexMappings() { { builder.startObject("_meta"); builder.field(TASK_RESULT_MAPPING_VERSION_META_FIELD, Version.CURRENT.toString()); - builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); // TODO[wrb]: constant + builder.field(SystemIndexDescriptor.VERSION_META_KEY, TASK_RESULTS_INDEX_MAPPINGS_VERSION); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index fa27602bbd9fa..0357492f2141f 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -251,7 +251,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng * This affects the system searchable snapshot cache index (not the searchable snapshot index itself) */ public static final String DATA_TIERS_CACHE_INDEX_PREFERENCE = String.join(",", DataTier.DATA_CONTENT, DataTier.DATA_HOT); - private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPING_VERSION = 0; + private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION = 0; private volatile Supplier repositoriesServiceSupplier; private final SetOnce blobStoreCacheService = new SetOnce<>(); @@ -614,7 +614,7 @@ private static XContentBuilder getIndexMappings() { { builder.startObject("_meta"); builder.field("version", Version.CURRENT); - builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCHABLE_SNAPSHOTS_INDEX_MAPPING_VERSION); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION); builder.endObject(); } { From 0fbb8fe3b7828a0a3fa8a05545da15b344b223b5 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 14:00:24 -0400 Subject: [PATCH 05/34] Fix up some tests --- .../alias/get/TransportGetAliasesActionTests.java | 2 +- .../create/TransportCreateIndexActionTests.java | 9 ++++++++- .../SystemIndexMetadataUpgradeServiceTests.java | 11 ++++++++++- .../core/security/test/TestRestrictedIndices.java | 3 ++- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java index 2919fd83a446f..bda31e3ebeeaa 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/alias/get/TransportGetAliasesActionTests.java @@ -274,7 +274,7 @@ public void testNetNewSystemIndicesDontErrorWhenNotRequested() { .setAliasName(".y") .setPrimaryIndex(".b") .setDescription(this.getTestName()) - .setMappings("{\"_meta\": {\"version\": \"1.0.0\"}}") + .setMappings("{\"_meta\": {\"version\": \"1.0.0\", \"system-index-mapping-version\": 0}}") .setSettings(Settings.EMPTY) .setVersionMetaKey("version") .setOrigin(this.getTestName()) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index 5ccb7900d7ff2..6359c928a5b70 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -63,7 +63,14 @@ public class TransportCreateIndexActionTests extends ESTestCase { .setPrimaryIndex(MANAGED_SYSTEM_INDEX_NAME + "-primary") .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) .setSettings(SystemIndexDescriptor.DEFAULT_SETTINGS) - .setMappings("{\"_meta\": {\"version\": \"1.0.0\"}}") + .setMappings(""" + { + "_meta": { + "version": "1.0.0", + "system-index-mapping-version": 0 + } + }" + """) .setVersionMetaKey("version") .setOrigin("origin") .build() diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java index 9b4307aae9c3e..4215c423da8c8 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java @@ -26,7 +26,16 @@ public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { - private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; + private static final String MAPPINGS = """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "system-index-mapping-version": 0 + } + } + } + """; private static final String SYSTEM_INDEX_NAME = ".myindex-1"; private static final String SYSTEM_ALIAS_NAME = ".myindex-alias"; private static final SystemIndexDescriptor DESCRIPTOR = SystemIndexDescriptor.builder() diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index 38bdb8ef18350..afa9ca5af4239 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -265,7 +265,8 @@ private static XContentBuilder mockMappings() { XContentBuilder builder = jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", Version.CURRENT) + .field("version", Version.CURRENT) // TODO[wrb]: remove + .field(SystemIndexDescriptor.VERSION_META_KEY, 0) .endObject() .field("dynamic", "strict") .startObject("properties") From 629b3f0a4646884f43e97e1c9201fe1732402ed9 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 14:13:36 -0400 Subject: [PATCH 06/34] 'mapping' -> 'mappings' for consistency --- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 2 +- .../indices/alias/get/TransportGetAliasesActionTests.java | 2 +- .../indices/create/TransportCreateIndexActionTests.java | 6 +++--- .../metadata/SystemIndexMetadataUpgradeServiceTests.java | 6 +++--- .../elasticsearch/indices/SystemIndexDescriptorTests.java | 6 +++--- .../src/main/resources/fleet-actions-results.json | 2 +- .../src/main/resources/fleet-actions.json | 2 +- .../template-resources/src/main/resources/fleet-agents.json | 2 +- .../src/main/resources/fleet-artifacts.json | 2 +- .../src/main/resources/fleet-enrollment-api-keys.json | 2 +- .../src/main/resources/fleet-policies-leader.json | 2 +- .../src/main/resources/fleet-policies.json | 2 +- .../src/main/resources/fleet-secrets.json | 2 +- .../src/main/resources/fleet-servers.json | 2 +- .../src/main/resources/ml/annotations_index_mappings.json | 2 +- .../src/main/resources/ml/config_index_mappings.json | 2 +- .../src/main/resources/ml/inference_index_mappings.json | 2 +- .../src/main/resources/ml/meta_index_mappings.json | 2 +- .../src/main/resources/ml/notifications_index_mappings.json | 2 +- .../src/main/resources/ml/stats_index_mappings.json | 2 +- 20 files changed, 26 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 7719797025e36..c09966b02f507 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -138,7 +138,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable Date: Tue, 25 Jul 2023 14:52:39 -0400 Subject: [PATCH 07/34] Set Fleet mappings versions --- .../TransportCreateIndexActionTests.java | 3 +- ...ystemIndexMetadataUpgradeServiceTests.java | 3 +- .../indices/SystemIndexDescriptorTests.java | 3 +- .../main/resources/fleet-actions-results.json | 4 +-- .../src/main/resources/fleet-actions.json | 4 +-- .../src/main/resources/fleet-agents.json | 4 +-- .../src/main/resources/fleet-artifacts.json | 4 +-- .../resources/fleet-enrollment-api-keys.json | 4 +-- .../main/resources/fleet-policies-leader.json | 4 +-- .../src/main/resources/fleet-policies.json | 4 +-- .../src/main/resources/fleet-secrets.json | 4 +-- .../src/main/resources/fleet-servers.json | 4 +-- .../org/elasticsearch/xpack/fleet/Fleet.java | 35 ++++++++++++------- 13 files changed, 47 insertions(+), 33 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java index ffae21c91accd..671bf13fb0e77 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexActionTests.java @@ -30,6 +30,7 @@ import org.mockito.ArgumentCaptor; import java.util.List; +import java.util.Locale; import static org.elasticsearch.cluster.metadata.IndexMetadata.SETTING_INDEX_HIDDEN; import static org.hamcrest.Matchers.equalTo; @@ -63,7 +64,7 @@ public class TransportCreateIndexActionTests extends ESTestCase { .setPrimaryIndex(MANAGED_SYSTEM_INDEX_NAME + "-primary") .setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED) .setSettings(SystemIndexDescriptor.DEFAULT_SETTINGS) - .setMappings(String.format(""" + .setMappings(String.format(Locale.ROOT, """ { "_meta": { "version": "1.0.0", diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java index 33f3dafd7eedc..bfdef69a5275a 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/SystemIndexMetadataUpgradeServiceTests.java @@ -19,6 +19,7 @@ import org.junit.Before; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.hamcrest.Matchers.equalTo; @@ -26,7 +27,7 @@ public class SystemIndexMetadataUpgradeServiceTests extends ESTestCase { - private static final String MAPPINGS = String.format(""" + private static final String MAPPINGS = String.format(Locale.ROOT, """ { "_doc": { "_meta": { diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 35c13dd5b1f05..63da869665ee0 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -19,6 +19,7 @@ import org.elasticsearch.xcontent.json.JsonXContent; import java.util.List; +import java.util.Locale; import java.util.Map; import static org.elasticsearch.indices.SystemIndexDescriptor.findDynamicMapping; @@ -28,7 +29,7 @@ public class SystemIndexDescriptorTests extends ESTestCase { // private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; - private static final String MAPPINGS = String.format(""" + private static final String MAPPINGS = String.format(Locale.ROOT, """ { "_doc": { "_meta": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index 999503c59e275..bbb9ccb2ea515 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -10,8 +10,8 @@ }, "mappings": { "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index b21ce56134a5a..71f260648552e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index c307ac76cfc9c..838a73ca7ba97 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index 5abd9109354de..017da19bb1848 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 62d904f73dd6e..7150f7043c811 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index 4492d2d078e87..b8ac5992f4a31 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index 7d609c41e72e4..caa7b1ef004e1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 58a3546a94a08..6120e2ccc1cb2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index 609e7b6879c3e..b61f445b89089 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "${fleet.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${fleet.version} }, "properties": { "agent": { diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index 95b3c86259aa2..ed36a998933ad 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -9,7 +9,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -79,6 +78,15 @@ public class Fleet extends Plugin implements SystemIndexPlugin { private static final String VERSION_KEY = "version"; private static final String MAPPING_VERSION_VARIABLE = "fleet.version"; private static final List ALLOWED_PRODUCTS = List.of("kibana", "fleet"); + private static final int FLEET_ACTIONS_MAPPINGS_VERSION = 1; + private static final int FLEET_AGENTS_MAPPINGS_VERSION = 1; + private static final int FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION = 1; + private static final int FLEET_SECRETS_MAPPINGS_VERSION = 1; + private static final int FLEET_POLICIES_MAPPINGS_VERSION = 1; + private static final int FLEET_POLICIES_LEADER_MAPPINGS_VERSION = 1; + private static final int FLEET_SERVERS_MAPPINGS_VERSION = 1; + private static final int FLEET_ARTIFACTS_MAPPINGS_VERSION = 1; + private static final int FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION = 1; @Override public Collection createComponents( @@ -139,7 +147,7 @@ public String getFeatureDescription() { private SystemIndexDescriptor fleetActionsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-actions.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-actions.json", FLEET_ACTIONS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -157,7 +165,7 @@ private SystemIndexDescriptor fleetActionsSystemIndexDescriptor() { private SystemIndexDescriptor fleetAgentsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-agents.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-agents.json", FLEET_AGENTS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -175,7 +183,10 @@ private SystemIndexDescriptor fleetAgentsSystemIndexDescriptor() { private SystemIndexDescriptor fleetEnrollmentApiKeysSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-enrollment-api-keys.json"), XContentType.JSON); + request.source( + loadTemplateSource("/fleet-enrollment-api-keys.json", FLEET_ENROLLMENT_API_KEYS_MAPPINGS_VERSION), + XContentType.JSON + ); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -193,7 +204,7 @@ private SystemIndexDescriptor fleetEnrollmentApiKeysSystemIndexDescriptor() { private SystemIndexDescriptor fleetSecretsSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-secrets.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-secrets.json", FLEET_SECRETS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) .setAllowedElasticProductOrigins(ALLOWED_PRODUCTS) @@ -210,7 +221,7 @@ private SystemIndexDescriptor fleetSecretsSystemIndexDescriptor() { private SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-policies.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-policies.json", FLEET_POLICIES_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -228,7 +239,7 @@ private SystemIndexDescriptor fleetPoliciesSystemIndexDescriptor() { private SystemIndexDescriptor fleetPoliciesLeaderSystemIndexDescriptor() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-policies-leader.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-policies-leader.json", FLEET_POLICIES_LEADER_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -246,7 +257,7 @@ private SystemIndexDescriptor fleetPoliciesLeaderSystemIndexDescriptor() { private SystemIndexDescriptor fleetServersSystemIndexDescriptors() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-servers.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-servers.json", FLEET_SERVERS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -264,7 +275,7 @@ private SystemIndexDescriptor fleetServersSystemIndexDescriptors() { private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { PutIndexTemplateRequest request = new PutIndexTemplateRequest(); - request.source(loadTemplateSource("/fleet-artifacts.json"), XContentType.JSON); + request.source(loadTemplateSource("/fleet-artifacts.json", FLEET_ARTIFACTS_MAPPINGS_VERSION), XContentType.JSON); return SystemIndexDescriptor.builder() .setType(Type.EXTERNAL_MANAGED) @@ -281,7 +292,7 @@ private SystemIndexDescriptor fleetArtifactsSystemIndexDescriptors() { } private SystemDataStreamDescriptor fleetActionsResultsDescriptor() { - final String source = loadTemplateSource("/fleet-actions-results.json"); + final String source = loadTemplateSource("/fleet-actions-results.json", FLEET_ACTIONS_RESULTS_MAPPINGS_VERSION); try (XContentParser parser = XContentType.JSON.xContent().createParser(XContentParserConfiguration.EMPTY, source)) { ComposableIndexTemplate composableIndexTemplate = ComposableIndexTemplate.parse(parser); return new SystemDataStreamDescriptor( @@ -336,8 +347,8 @@ public void cleanUpFeature(ClusterService clusterService, Client client, ActionL } } - private String loadTemplateSource(String resource) { - return TemplateUtils.loadTemplate(resource, Version.CURRENT.toString(), MAPPING_VERSION_VARIABLE); + private String loadTemplateSource(String resource, int mappingsVersion) { + return TemplateUtils.loadTemplate(resource, Integer.toString(mappingsVersion), MAPPING_VERSION_VARIABLE); } @Override From 9074fd5d6a9c58b2d662c78f08b53d9130174fe4 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 16:01:21 -0400 Subject: [PATCH 08/34] Update test --- .../admin/indices/resolve/ResolveIndexTests.java | 12 +++++++++++- .../elasticsearch/xpack/core/ml/MlConfigIndex.java | 8 ++++++-- .../org/elasticsearch/xpack/core/ml/MlMetaIndex.java | 8 ++++++-- .../elasticsearch/xpack/core/ml/MlStatsIndex.java | 8 ++++++-- .../xpack/core/ml/annotations/AnnotationIndex.java | 8 ++++++-- .../persistence/InferenceIndexConstants.java | 8 ++++++-- .../core/ml/notifications/NotificationsIndex.java | 4 ++-- .../resources/ml/annotations_index_mappings.json | 4 ++-- .../src/main/resources/ml/config_index_mappings.json | 4 ++-- .../main/resources/ml/inference_index_mappings.json | 4 ++-- .../src/main/resources/ml/meta_index_mappings.json | 4 ++-- .../resources/ml/notifications_index_mappings.json | 4 ++-- .../src/main/resources/ml/stats_index_mappings.json | 4 ++-- 13 files changed, 55 insertions(+), 25 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java index 11b919416c352..64f272c5a071c 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexTests.java @@ -44,6 +44,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -513,7 +514,16 @@ private SystemIndices addSystemIndex(Metadata.Builder mdBuilder) { .setAllowedElasticProductOrigins(List.of("test-net-new-system")) .setNetNew() .setSettings(Settings.EMPTY) - .setMappings("{ \"_doc\": { \"_meta\": { \"version\": \"8.0.0\" } } }") + .setMappings(String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "8.0.0", + "%s": 1 + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY)) .setPrimaryIndex(".test-net-new-system-1") .setVersionMetaKey("version") .setOrigin("system") diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 2499734672053..e4ddcea657225 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; @@ -18,6 +17,7 @@ public final class MlConfigIndex { private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; + private static final int CONFIG_INDEX_MAPPINGS_VERSION = 1; /** * The name of the index where job, datafeed and analytics configuration is stored @@ -29,7 +29,11 @@ public static String indexName() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/config_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/config_index_mappings.json", + Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION), + MAPPINGS_VERSION_VARIABLE + ); } public static Settings settings() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index a72a051414677..e20bd4bd96441 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.template.TemplateUtils; @@ -15,6 +14,7 @@ public final class MlMetaIndex { private static final String INDEX_NAME = ".ml-meta"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int META_INDEX_MAPPINGS_VERSION = 1; /** * Where to store the ml info in Elasticsearch - must match what's @@ -27,7 +27,11 @@ public static String indexName() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/meta_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/meta_index_mappings.json", + Integer.toString(META_INDEX_MAPPINGS_VERSION), + MAPPINGS_VERSION_VARIABLE + ); } public static Settings settings() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java index 76234954b4107..a732d530d301b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -25,6 +24,7 @@ public class MlStatsIndex { public static final String TEMPLATE_NAME = ".ml-stats"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int STATS_INDEX_MAPPINGS_VERSION = 1; private MlStatsIndex() {} @@ -36,7 +36,11 @@ public static String wrappedMapping() { } public static String mapping() { - return TemplateUtils.loadTemplate("/ml/stats_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/stats_index_mappings.json", + Integer.toString(STATS_INDEX_MAPPINGS_VERSION), + MAPPINGS_VERSION_VARIABLE + ); } public static String indexPattern() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index c509e52675e38..251b9c872608d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -56,6 +55,7 @@ public class AnnotationIndex { public static final List OLD_INDEX_NAMES = List.of(".ml-annotations-6"); private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 1; /** * Create the .ml-annotations-6 index with correct mappings if it does not already exist. This index is read and written by the UI @@ -208,6 +208,10 @@ public static void createAnnotationsIndexIfNecessary( } public static String annotationsMapping() { - return TemplateUtils.loadTemplate("/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/annotations_index_mappings.json", + Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION), + MAPPINGS_VERSION_VARIABLE + ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index aa8fa7c88e719..7b78d5f39982b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml.inference.persistence; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xcontent.ParseField; @@ -44,9 +43,14 @@ public final class InferenceIndexConstants { private static final String NATIVE_LATEST_INDEX = NATIVE_INDEX_PREFIX + NATIVE_INDEX_VERSION; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int INFERENCE_INDEX_MAPPINGS_VERSION = 1; public static String mapping() { - return TemplateUtils.loadTemplate("/ml/inference_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + "/ml/inference_index_mappings.json", + Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION), + MAPPINGS_VERSION_VARIABLE + ); } public static String nativeDefinitionStore() { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 11c4ec2aa0b77..24615f390375c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.core.ml.notifications; -import org.elasticsearch.Version; import org.elasticsearch.xpack.core.template.TemplateUtils; public final class NotificationsIndex { @@ -15,13 +14,14 @@ public final class NotificationsIndex { private static final String RESOURCE_PATH = "/ml/"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; + private static final int NOTIFICATIONS_INDEX_MAPPINGS_VERSION = 1; private NotificationsIndex() {} public static String mapping() { return TemplateUtils.loadTemplate( RESOURCE_PATH + "notifications_index_mappings.json", - Version.CURRENT.toString(), + Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION), MAPPINGS_VERSION_VARIABLE ); } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index eb7a561dabb17..db5cb6fc1cf21 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta" : { - "version" : "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index 28736312e4509..f1b145c3066db 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -1,8 +1,8 @@ { "_doc" : { "_meta" : { - "version" : "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index 300aa5e344aea..e1d2846551362 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index 721f3de604df6..f1221c6f50868 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta": { - "version": "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index 7adc86c3f0c50..b5e92efbc75c3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,7 +1,7 @@ { "_meta" : { - "version" : "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index 3560a364b0fc9..a4d8af421cdca 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,7 +1,7 @@ { "_meta": { - "version" : "${xpack.ml.version}", - "system-index-mappings-version": 0 + "version": "8.99.99", + "system-index-mappings-version": ${xpack.ml.version} }, "dynamic": false, "properties" : { From 6c094c7557754c40e59c1019746aa69e8a8491b9 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 25 Jul 2023 16:33:59 -0400 Subject: [PATCH 09/34] Bump starting versions and use placeholder for old versions --- .../elasticsearch/ingest/geoip/IngestGeoIpPlugin.java | 5 ++--- .../synonyms/SynonymsManagementAPIService.java | 5 ++--- .../java/org/elasticsearch/tasks/TaskResultsService.java | 3 +-- .../xpack/core/async/AsyncTaskIndexService.java | 3 +-- .../xpack/core/security/test/TestRestrictedIndices.java | 3 +-- .../xpack/application/rules/QueryRulesIndexService.java | 5 ++--- .../search/SearchApplicationIndexService.java | 5 ++--- .../java/org/elasticsearch/xpack/logstash/Logstash.java | 5 ++--- .../xpack/searchablesnapshots/SearchableSnapshots.java | 5 ++--- .../xpack/security/support/SecuritySystemIndices.java | 6 +++--- .../transform/persistence/TransformInternalIndex.java | 5 ++--- .../java/org/elasticsearch/xpack/watcher/Watcher.java | 9 ++++----- 12 files changed, 24 insertions(+), 35 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index c15a8dd1fd4a7..828d6d32499c9 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -9,7 +9,6 @@ package org.elasticsearch.ingest.geoip; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; @@ -73,7 +72,7 @@ public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin { public static final Setting CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope); - private static final int GEOIP_INDEX_MAPPINGS_VERSION = 0; + private static final int GEOIP_INDEX_MAPPINGS_VERSION = 1; static Set DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb"); @@ -225,7 +224,7 @@ private static XContentBuilder mappings() { return jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", Version.CURRENT) + .field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION) .field(SystemIndexDescriptor.VERSION_META_KEY, GEOIP_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") diff --git a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java index 5ea60dceeb65e..2b36f45f90ed6 100644 --- a/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java +++ b/server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java @@ -11,7 +11,6 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -81,7 +80,7 @@ public class SynonymsManagementAPIService { public static final int MAX_SYNONYMS_SETS = 10_000; private static final String SYNONYM_RULE_ID_FIELD = SynonymRule.ID_FIELD.getPreferredName(); private static final String SYNONYM_SETS_AGG_NAME = "synonym_sets_aggr"; - private static final int SYNONYMS_INDEX_MAPPINGS_VERSION = 0; + private static final int SYNONYMS_INDEX_MAPPINGS_VERSION = 1; private final Client client; @@ -123,7 +122,7 @@ private static XContentBuilder mappings() { { builder.startObject("_meta"); { - builder.field("version", Version.CURRENT.toString()); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, SYNONYMS_INDEX_MAPPINGS_VERSION); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java index 1e42417aa2471..f6f34949558bc 100644 --- a/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java +++ b/server/src/main/java/org/elasticsearch/tasks/TaskResultsService.java @@ -10,7 +10,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.bulk.BackoffPolicy; import org.elasticsearch.action.index.IndexRequestBuilder; @@ -124,7 +123,7 @@ private static XContentBuilder getTaskResultIndexMappings() { builder.startObject(); { builder.startObject("_meta"); - builder.field(TASK_RESULT_MAPPING_VERSION_META_FIELD, Version.CURRENT.toString()); + builder.field(TASK_RESULT_MAPPING_VERSION_META_FIELD, SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, TASK_RESULTS_INDEX_MAPPINGS_VERSION); builder.endObject(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java index be6c1a29bcc6f..35427e749cb93 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/async/AsyncTaskIndexService.java @@ -12,7 +12,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.delete.DeleteRequest; import org.elasticsearch.action.delete.DeleteResponse; @@ -110,7 +109,7 @@ private static XContentBuilder mappings() { XContentBuilder builder = jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", Version.CURRENT) + .field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION) .field(SystemIndexDescriptor.VERSION_META_KEY, ASYNC_TASK_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index afa9ca5af4239..5d86ab5a9f01c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.core.security.test; -import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.ComposableIndexTemplate; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -265,7 +264,7 @@ private static XContentBuilder mockMappings() { XContentBuilder builder = jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", Version.CURRENT) // TODO[wrb]: remove + .field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION) // TODO[wrb]: remove .field(SystemIndexDescriptor.VERSION_META_KEY, 0) .endObject() .field("dynamic", "strict") diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java index d3a091e7a2fac..6e428ec10a776 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/rules/QueryRulesIndexService.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.application.rules; import org.elasticsearch.ResourceNotFoundException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -62,7 +61,7 @@ public class QueryRulesIndexService { public static final String QUERY_RULES_ALIAS_NAME = ".query-rules"; public static final String QUERY_RULES_CONCRETE_INDEX_NAME = ".query-rules-1"; public static final String QUERY_RULES_INDEX_NAME_PATTERN = ".query-rules-*"; - private static final int QUERY_RULES_INDEX_MAPPINGS_VERSION = 0; + private static final int QUERY_RULES_INDEX_MAPPINGS_VERSION = 1; private final Client clientWithOrigin; private final ClusterSettings clusterSettings; @@ -106,7 +105,7 @@ private static XContentBuilder getIndexMappings() { builder.startObject(); { builder.startObject("_meta"); - builder.field("version", Version.CURRENT.toString()); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, QUERY_RULES_INDEX_MAPPINGS_VERSION); builder.endObject(); diff --git a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java index 165d2309f8ae0..66d15db65b660 100644 --- a/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java +++ b/x-pack/plugin/ent-search/src/main/java/org/elasticsearch/xpack/application/search/SearchApplicationIndexService.java @@ -11,7 +11,6 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; import org.elasticsearch.TransportVersion; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DelegatingActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -95,7 +94,7 @@ public class SearchApplicationIndexService { public static final String SEARCH_APPLICATION_ALIAS_NAME = ".search-app"; public static final String SEARCH_APPLICATION_CONCRETE_INDEX_NAME = ".search-app-1"; public static final String SEARCH_APPLICATION_INDEX_NAME_PATTERN = ".search-app-*"; - private static final int SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION = 0; + private static final int SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION = 1; private static final String INCONSISTENT_INDICES_NOT_IN_ALIAS_MSG = "index is in search application but not in associated alias"; private static final String INCONSISTENT_ALIAS_NOT_IN_INDICES_MSG = "index is in alias but not associcated with search application"; @@ -155,7 +154,7 @@ private static XContentBuilder getIndexMappings() { builder.startObject(); { builder.startObject("_meta"); - builder.field("version", Version.CURRENT.toString()); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCH_APPLICATION_INDEX_MAPPINGS_VERSION); builder.endObject(); diff --git a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java index 4fe588e039e26..995888824a137 100644 --- a/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java +++ b/x-pack/plugin/logstash/src/main/java/org/elasticsearch/xpack/logstash/Logstash.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.xpack.logstash; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexMetadata; @@ -58,7 +57,7 @@ public class Logstash extends Plugin implements SystemIndexPlugin { public static final String LOGSTASH_CONCRETE_INDEX_NAME = ".logstash"; public static final String LOGSTASH_INDEX_NAME_PATTERN = LOGSTASH_CONCRETE_INDEX_NAME + "*"; - public static final int LOGSTASH_INDEX_MAPPINGS_VERSION = 0; + public static final int LOGSTASH_INDEX_MAPPINGS_VERSION = 1; static final LicensedFeature.Momentary LOGSTASH_FEATURE = LicensedFeature.momentary(null, "logstash", License.OperationMode.STANDARD); @@ -121,7 +120,7 @@ private XContentBuilder getIndexMappings() { builder.field("dynamic", "strict"); { builder.startObject("_meta"); - builder.field("logstash-version", Version.CURRENT); + builder.field("logstash-version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, LOGSTASH_INDEX_MAPPINGS_VERSION); builder.endObject(); } diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java index 0357492f2141f..88271bbd84f43 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/SearchableSnapshots.java @@ -8,7 +8,6 @@ import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.util.SetOnce; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -251,7 +250,7 @@ public class SearchableSnapshots extends Plugin implements IndexStorePlugin, Eng * This affects the system searchable snapshot cache index (not the searchable snapshot index itself) */ public static final String DATA_TIERS_CACHE_INDEX_PREFERENCE = String.join(",", DataTier.DATA_CONTENT, DataTier.DATA_HOT); - private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION = 0; + private static final int SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION = 1; private volatile Supplier repositoriesServiceSupplier; private final SetOnce blobStoreCacheService = new SetOnce<>(); @@ -613,7 +612,7 @@ private static XContentBuilder getIndexMappings() { builder.field("dynamic", "strict"); { builder.startObject("_meta"); - builder.field("version", Version.CURRENT); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, SEARCHABLE_SNAPSHOTS_INDEX_MAPPINGS_VERSION); builder.endObject(); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java index 5e931444f04cd..c4648048ac72a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecuritySystemIndices.java @@ -36,11 +36,11 @@ public class SecuritySystemIndices { public static final int INTERNAL_MAIN_INDEX_FORMAT = 6; - private static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 0; + private static final int INTERNAL_MAIN_INDEX_MAPPINGS_FORMAT = 1; private static final int INTERNAL_TOKENS_INDEX_FORMAT = 7; - private static final int INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT = 0; + private static final int INTERNAL_TOKENS_INDEX_MAPPINGS_FORMAT = 1; private static final int INTERNAL_PROFILE_INDEX_FORMAT = 8; - private static final int INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT = 0; + private static final int INTERNAL_PROFILE_INDEX_MAPPINGS_FORMAT = 1; public static final String SECURITY_MAIN_ALIAS = ".security"; private static final String MAIN_INDEX_CONCRETE_NAME = ".security-7"; diff --git a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java index 14ca822d5a493..c8a130c4cd52b 100644 --- a/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java +++ b/x-pack/plugin/transform/src/main/java/org/elasticsearch/xpack/transform/persistence/TransformInternalIndex.java @@ -8,7 +8,6 @@ package org.elasticsearch.xpack.transform.persistence; import org.elasticsearch.ResourceAlreadyExistsException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse; @@ -87,7 +86,7 @@ public final class TransformInternalIndex { public static final String KEYWORD = "keyword"; public static final String BOOLEAN = "boolean"; public static final String FLATTENED = "flattened"; - public static final int TRANSFORM_INDEX_MAPPINGS_VERSION = 0; + public static final int TRANSFORM_INDEX_MAPPINGS_VERSION = 1; public static SystemIndexDescriptor getSystemIndexDescriptor(Settings transformInternalIndexAdditionalSettings) throws IOException { return SystemIndexDescriptor.builder() @@ -359,7 +358,7 @@ private static XContentBuilder addTransformCheckpointMappings(XContentBuilder bu */ private static XContentBuilder addMetaInformation(XContentBuilder builder) throws IOException { return builder.startObject("_meta") - .field("version", Version.CURRENT) + .field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION) .field(SystemIndexDescriptor.VERSION_META_KEY, TRANSFORM_INDEX_MAPPINGS_VERSION) .endObject(); } diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java index 957adf2bd40f2..7d24594801e8b 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/Watcher.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -276,8 +275,8 @@ public class Watcher extends Plugin implements SystemIndexPlugin, ScriptPlugin, ); private static final Logger logger = LogManager.getLogger(Watcher.class); - private static final int WATCHES_INDEX_MAPPINGS_VERSION = 0; - private static final int TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION = 0; + private static final int WATCHES_INDEX_MAPPINGS_VERSION = 1; + private static final int TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION = 1; private WatcherIndexingListener listener; private HttpClient httpClient; private BulkProcessor2 bulkProcessor; @@ -877,7 +876,7 @@ private XContentBuilder getWatchesIndexMappings() { builder.field("dynamic", "strict"); { builder.startObject("_meta"); - builder.field("version", Version.CURRENT); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } @@ -969,7 +968,7 @@ private XContentBuilder getTriggeredWatchesIndexMappings() { builder.field("dynamic", "strict"); { builder.startObject("_meta"); - builder.field("version", Version.CURRENT); + builder.field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); builder.field(SystemIndexDescriptor.VERSION_META_KEY, TRIGGERED_WATCHES_INDEX_MAPPINGS_VERSION); builder.endObject(); } From d522b26f5c33ce028a9ea8cb4e4133b0ccb1356f Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 26 Jul 2023 15:17:28 -0400 Subject: [PATCH 10/34] Add versions for a few more test descriptors --- .../migration/AbstractFeatureMigrationIntegTest.java | 1 + .../org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java | 1 + .../org/elasticsearch/indices/TestSystemIndexDescriptor.java | 2 ++ 3 files changed, 4 insertions(+) diff --git a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java index 6f9bcbfbf2a4e..1955cf25d6436 100644 --- a/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java +++ b/modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java @@ -204,6 +204,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna { builder.startObject("_meta"); builder.field(VERSION_META_KEY, META_VERSION); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.field(DESCRIPTOR_MANAGED_META_KEY, descriptorManaged); builder.field(DESCRIPTOR_INTERNAL_META_KEY, descriptorInternal); builder.endObject(); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java index 9f2a9d3a48162..105e9f5ec91f0 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java @@ -68,6 +68,7 @@ public Collection getSystemIndexDescriptors(Settings sett { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.endObject(); builder.field("dynamic", "strict"); diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java index 58830f978ee30..0a090fa889a29 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java @@ -97,6 +97,7 @@ public static String getOldMappings() { builder.startObject(); { builder.startObject("_meta"); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0); builder.field("version", Version.CURRENT.previousMajor().toString()); builder.endObject(); @@ -123,6 +124,7 @@ public static String getNewMappings() { builder.startObject(); { builder.startObject("_meta"); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.field("version", Version.CURRENT.toString()); builder.endObject(); From 3adac843201dd20e3eb1bc51969a2d67e0637fe3 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 26 Jul 2023 15:46:08 -0400 Subject: [PATCH 11/34] Try fixing ML mapping tests --- .../xpack/core/ml/MlConfigIndex.java | 2 +- .../core/ml/annotations/AnnotationIndex.java | 2 +- .../persistence/InferenceIndexConstants.java | 2 +- .../upgrades/MlMappingsUpgradeIT.java | 16 +++++++++++++--- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index e4ddcea657225..26838b3521baf 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -17,7 +17,7 @@ public final class MlConfigIndex { private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; - private static final int CONFIG_INDEX_MAPPINGS_VERSION = 1; + public static final int CONFIG_INDEX_MAPPINGS_VERSION = 2; /** * The name of the index where job, datafeed and analytics configuration is stored diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 251b9c872608d..ff199a1bd17a5 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -55,7 +55,7 @@ public class AnnotationIndex { public static final List OLD_INDEX_NAMES = List.of(".ml-annotations-6"); private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - private static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 1; + public static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 3; /** * Create the .ml-annotations-6 index with correct mappings if it does not already exist. This index is read and written by the UI diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index 7b78d5f39982b..017b63196e794 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -43,7 +43,7 @@ public final class InferenceIndexConstants { private static final String NATIVE_LATEST_INDEX = NATIVE_INDEX_PREFIX + NATIVE_INDEX_VERSION; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - private static final int INFERENCE_INDEX_MAPPINGS_VERSION = 1; + public static final int INFERENCE_INDEX_MAPPINGS_VERSION = 4; public static String mapping() { return TemplateUtils.loadTemplate( diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 39efc221ed129..b3809f9a7ad97 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -10,6 +10,10 @@ import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; +import org.elasticsearch.indices.SystemIndexDescriptor; +import org.elasticsearch.xpack.core.ml.MlConfigIndex; +import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; +import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; @@ -128,7 +132,9 @@ private void assertUpgradedResultsMappings() throws Exception { } assertNotNull(indexLevel); - assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(InferenceIndexConstants.INFERENCE_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", + indexLevel)); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword" @@ -161,7 +167,9 @@ private void assertUpgradedAnnotationsMappings() throws Exception { } assertNotNull(indexLevel); - assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", + indexLevel)); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that would be incorrectly mapped by dynamic @@ -215,7 +223,9 @@ private void assertUpgradedConfigMappings() throws Exception { Map indexLevel = (Map) responseLevel.get(".ml-config"); assertNotNull(indexLevel); - assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); + assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", + indexLevel)); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword" From 2d0f8068d0c32c31de3c21989e71d127053cab40 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Wed, 26 Jul 2023 17:22:27 -0400 Subject: [PATCH 12/34] spotless --- .../upgrades/MlMappingsUpgradeIT.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index b3809f9a7ad97..878b894224afe 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -6,7 +6,6 @@ */ package org.elasticsearch.upgrades; -import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -133,8 +132,10 @@ private void assertUpgradedResultsMappings() throws Exception { assertNotNull(indexLevel); assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); - assertEquals(InferenceIndexConstants.INFERENCE_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", - indexLevel)); + assertEquals( + InferenceIndexConstants.INFERENCE_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.system-index-mappings-version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword" @@ -168,8 +169,10 @@ private void assertUpgradedAnnotationsMappings() throws Exception { assertNotNull(indexLevel); assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); - assertEquals(AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", - indexLevel)); + assertEquals( + AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.system-index-mappings-version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that would be incorrectly mapped by dynamic @@ -224,8 +227,10 @@ private void assertUpgradedConfigMappings() throws Exception { assertNotNull(indexLevel); assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); - assertEquals(MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", - indexLevel)); + assertEquals( + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, + extractValue("mappings._meta.system-index-mappings-version", indexLevel) + ); // TODO: as the years go by, the field we assert on here should be changed // to the most recent field we've added that is NOT of type "keyword" From 04ac9e952b4ff67cbfd65e5da6c0ec050725f5aa Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 10:19:47 -0400 Subject: [PATCH 13/34] Fix watcher tests --- .../xpack/restart/WatcherMappingUpdateIT.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java index 043b3f49a8825..5e92d211af428 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/WatcherMappingUpdateIT.java @@ -16,6 +16,7 @@ import org.elasticsearch.client.Response; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.upgrades.FullClusterRestartUpgradeStatus; import java.nio.charset.StandardCharsets; @@ -63,17 +64,17 @@ public void testMappingsAreUpdated() throws Exception { client().performRequest(putWatchRequest); if (getOldClusterVersion().onOrAfter(Version.V_7_13_0)) { - assertMappingVersion(".watches", getOldClusterVersion()); + assertMappingVersion(".watches", getOldClusterVersion().toString()); } else { // watches indices from before 7.10 do not have mapping versions in _meta assertNoMappingVersion(".watches"); } } else { - assertMappingVersion(".watches", Version.CURRENT); + assertMappingVersion(".watches", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION); } } - private void assertMappingVersion(String index, Version clusterVersion) throws Exception { + private void assertMappingVersion(String index, String clusterVersion) throws Exception { assertBusy(() -> { Request mappingRequest = new Request("GET", index + "/_mappings"); mappingRequest.setOptions(getWarningHandlerOptions(index)); From 59352c5ac3268aafd9a7ba56d805f0930502e2b0 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 11:17:19 -0400 Subject: [PATCH 14/34] Back out 'legacy mapping' constant --- .../org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java | 3 ++- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 4 ---- .../synonyms/SynonymsManagementAPIService.java | 3 ++- .../java/org/elasticsearch/tasks/TaskResultsService.java | 3 ++- .../xpack/core/async/AsyncTaskIndexService.java | 3 ++- .../xpack/core/security/test/TestRestrictedIndices.java | 3 ++- .../xpack/application/rules/QueryRulesIndexService.java | 3 ++- .../application/search/SearchApplicationIndexService.java | 3 ++- .../java/org/elasticsearch/xpack/logstash/Logstash.java | 3 ++- .../xpack/searchablesnapshots/SearchableSnapshots.java | 3 ++- .../transform/persistence/TransformInternalIndex.java | 3 ++- .../java/org/elasticsearch/xpack/watcher/Watcher.java | 5 +++-- .../xpack/restart/WatcherMappingUpdateIT.java | 7 +++---- .../org/elasticsearch/upgrades/MlMappingsUpgradeIT.java | 8 ++++---- 14 files changed, 30 insertions(+), 24 deletions(-) diff --git a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java index 828d6d32499c9..336e650717952 100644 --- a/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java +++ b/modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java @@ -9,6 +9,7 @@ package org.elasticsearch.ingest.geoip; import org.apache.lucene.util.SetOnce; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.client.internal.Client; @@ -224,7 +225,7 @@ private static XContentBuilder mappings() { return jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION) + .field("version", Version.CURRENT) .field(SystemIndexDescriptor.VERSION_META_KEY, GEOIP_INDEX_MAPPINGS_VERSION) .endObject() .field("dynamic", "strict") diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index c09966b02f507..93bc88d514768 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -140,10 +140,6 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable { Request mappingRequest = new Request("GET", index + "/_mappings"); mappingRequest.setOptions(getWarningHandlerOptions(index)); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 878b894224afe..3cf216d9f8de9 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -6,10 +6,10 @@ */ package org.elasticsearch.upgrades; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; -import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; @@ -131,7 +131,7 @@ private void assertUpgradedResultsMappings() throws Exception { } assertNotNull(indexLevel); - assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( InferenceIndexConstants.INFERENCE_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", indexLevel) @@ -168,7 +168,7 @@ private void assertUpgradedAnnotationsMappings() throws Exception { } assertNotNull(indexLevel); - assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", indexLevel) @@ -226,7 +226,7 @@ private void assertUpgradedConfigMappings() throws Exception { Map indexLevel = (Map) responseLevel.get(".ml-config"); assertNotNull(indexLevel); - assertEquals(SystemIndexDescriptor.LEGACY_PLACEHOLDER_VERSION, extractValue("mappings._meta.version", indexLevel)); + assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", indexLevel) From e47b89c23a419d3a8d372a8b1d1ec76c406121d9 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 11:23:10 -0400 Subject: [PATCH 15/34] Fix mappings resources --- .../src/main/resources/fleet-actions-results.json | 4 ++-- .../template-resources/src/main/resources/fleet-actions.json | 4 ++-- .../template-resources/src/main/resources/fleet-agents.json | 4 ++-- .../src/main/resources/fleet-artifacts.json | 4 ++-- .../src/main/resources/fleet-enrollment-api-keys.json | 4 ++-- .../src/main/resources/fleet-policies-leader.json | 4 ++-- .../template-resources/src/main/resources/fleet-policies.json | 4 ++-- .../template-resources/src/main/resources/fleet-secrets.json | 4 ++-- .../template-resources/src/main/resources/fleet-servers.json | 4 ++-- .../src/main/resources/ml/annotations_index_mappings.json | 4 ++-- .../src/main/resources/ml/config_index_mappings.json | 4 ++-- .../src/main/resources/ml/inference_index_mappings.json | 4 ++-- .../src/main/resources/ml/meta_index_mappings.json | 4 ++-- .../src/main/resources/ml/notifications_index_mappings.json | 4 ++-- .../src/main/resources/ml/stats_index_mappings.json | 4 ++-- 15 files changed, 30 insertions(+), 30 deletions(-) diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index bbb9ccb2ea515..7ad0ff9174c7f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -10,8 +10,8 @@ }, "mappings": { "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index 71f260648552e..3f3868155153f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 838a73ca7ba97..3e6784e8e4d01 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index 017da19bb1848..43bbf934c74f8 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 7150f7043c811..1c8c8df24f8b3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index b8ac5992f4a31..9c7ff64abd4e2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index caa7b1ef004e1..82427ff3bfec1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 6120e2ccc1cb2..0745982b12986 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index b61f445b89089..694674f8cdeb0 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -6,8 +6,8 @@ "_doc" : { "dynamic": false, "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${fleet.version} + "version": "${fleet.version}", + "system-index-mappings-version": 1 }, "properties": { "agent": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index db5cb6fc1cf21..81c95b063acd1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta" : { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version}", + "system-index-mappings-version": 1 }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index f1b145c3066db..0e48b8f427b43 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -1,8 +1,8 @@ { "_doc" : { "_meta" : { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version", + "system-index-mappings-version": 1 }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index e1d2846551362..3b5cf5f91dec9 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version}", + "system-index-mappings-version": 1 }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index f1221c6f50868..dc52d538c09d1 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -1,8 +1,8 @@ { "_doc": { "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version}", + "system-index-mappings-version": 1 }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index b5e92efbc75c3..42087ad363c84 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,7 +1,7 @@ { "_meta" : { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version}", + "system-index-mappings-version": 1 }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index a4d8af421cdca..cf82f52aac9ee 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,7 +1,7 @@ { "_meta": { - "version": "8.99.99", - "system-index-mappings-version": ${xpack.ml.version} + "version": "${xpack.ml.version}", + "system-index-mappings-version": 1 }, "dynamic": false, "properties" : { From d56c1b70f30a30bce96edc991fca88038a3774a2 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 11:26:31 -0400 Subject: [PATCH 16/34] Add version int to qa test --- .../java/org/elasticsearch/system/indices/SystemIndicesQA.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java index f55b350520106..d4b242e613c95 100644 --- a/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java +++ b/qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java @@ -111,6 +111,7 @@ private static XContentBuilder mappings() { .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") .field("version", Version.CURRENT) + .field(SystemIndexDescriptor.VERSION_META_KEY, 1) .endObject() .field("dynamic", "strict") .startObject("properties") From 76ab8accf383e2bf9e81fa1f23627b98e618808e Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 11:28:08 -0400 Subject: [PATCH 17/34] Add version int to system index rest test --- .../java/org/elasticsearch/http/SystemIndexRestIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java index 9847534c1c042..7d0095dd81f63 100644 --- a/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java +++ b/qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java @@ -169,6 +169,7 @@ public Collection getSystemIndexDescriptors(Settings sett { builder.startObject("_meta"); builder.field("version", Version.CURRENT.toString()); + builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1); builder.endObject(); builder.field("dynamic", "strict"); From 01851120f0117196496635ba43bec22704450317 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 11:59:59 -0400 Subject: [PATCH 18/34] Insert new version into templates --- .../org/elasticsearch/xpack/core/ml/MlConfigIndex.java | 8 ++++++-- .../java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java | 8 ++++++-- .../org/elasticsearch/xpack/core/ml/MlStatsIndex.java | 7 +++++-- .../xpack/core/ml/annotations/AnnotationIndex.java | 7 +++++-- .../ml/inference/persistence/InferenceIndexConstants.java | 8 ++++++-- .../xpack/core/ml/notifications/NotificationsIndex.java | 8 ++++++-- .../src/main/resources/fleet-actions-results.json | 2 +- .../src/main/resources/fleet-actions.json | 2 +- .../src/main/resources/fleet-agents.json | 2 +- .../src/main/resources/fleet-artifacts.json | 2 +- .../src/main/resources/fleet-enrollment-api-keys.json | 2 +- .../src/main/resources/fleet-policies-leader.json | 2 +- .../src/main/resources/fleet-policies.json | 2 +- .../src/main/resources/fleet-secrets.json | 2 +- .../src/main/resources/fleet-servers.json | 2 +- .../src/main/resources/ml/annotations_index_mappings.json | 2 +- .../src/main/resources/ml/config_index_mappings.json | 4 ++-- .../src/main/resources/ml/inference_index_mappings.json | 2 +- .../src/main/resources/ml/meta_index_mappings.json | 2 +- .../main/resources/ml/notifications_index_mappings.json | 2 +- .../src/main/resources/ml/stats_index_mappings.json | 2 +- .../main/java/org/elasticsearch/xpack/fleet/Fleet.java | 8 +++++++- 22 files changed, 57 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 26838b3521baf..62af62cd74ff9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -6,11 +6,14 @@ */ package org.elasticsearch.xpack.core.ml; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class MlConfigIndex { private static final String INDEX_NAME = ".ml-config"; @@ -31,8 +34,9 @@ public static String indexName() { public static String mapping() { return TemplateUtils.loadTemplate( "/ml/config_index_mappings.json", - Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index e20bd4bd96441..04c4b26bc9e95 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -6,10 +6,13 @@ */ package org.elasticsearch.xpack.core.ml; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class MlMetaIndex { private static final String INDEX_NAME = ".ml-meta"; @@ -29,8 +32,9 @@ public static String indexName() { public static String mapping() { return TemplateUtils.loadTemplate( "/ml/meta_index_mappings.json", - Integer.toString(META_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(META_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java index a732d530d301b..10549f42f5a8a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.core.ml; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.client.internal.Client; import org.elasticsearch.cluster.ClusterState; @@ -15,6 +16,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Locale; +import java.util.Map; /** * Describes the indices where ML is storing various stats about the users jobs. @@ -38,8 +40,9 @@ public static String wrappedMapping() { public static String mapping() { return TemplateUtils.loadTemplate( "/ml/stats_index_mappings.json", - Integer.toString(STATS_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(STATS_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index ff199a1bd17a5..79949f5d4b141 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ResourceAlreadyExistsException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.admin.cluster.health.ClusterHealthAction; import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest; @@ -30,6 +31,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.List; +import java.util.Map; import java.util.SortedMap; import static java.lang.Thread.currentThread; @@ -210,8 +212,9 @@ public static void createAnnotationsIndexIfNecessary( public static String annotationsMapping() { return TemplateUtils.loadTemplate( "/ml/annotations_index_mappings.json", - Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index 017b63196e794..3f9e3b515ee4b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -6,11 +6,14 @@ */ package org.elasticsearch.xpack.core.ml.inference.persistence; +import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.xcontent.ParseField; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + /** * Class containing the index constants so that the index version, name, and prefix are available to a wider audience. */ @@ -48,8 +51,9 @@ public final class InferenceIndexConstants { public static String mapping() { return TemplateUtils.loadTemplate( "/ml/inference_index_mappings.json", - Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 24615f390375c..18ecf1379c10a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -6,8 +6,11 @@ */ package org.elasticsearch.xpack.core.ml.notifications; +import org.elasticsearch.Version; import org.elasticsearch.xpack.core.template.TemplateUtils; +import java.util.Map; + public final class NotificationsIndex { public static final String NOTIFICATIONS_INDEX = ".ml-notifications-000002"; @@ -21,8 +24,9 @@ private NotificationsIndex() {} public static String mapping() { return TemplateUtils.loadTemplate( RESOURCE_PATH + "notifications_index_mappings.json", - Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION), - MAPPINGS_VERSION_VARIABLE + Version.CURRENT.toString(), + MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index 7ad0ff9174c7f..d04708d26502a 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -11,7 +11,7 @@ "mappings": { "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index 3f3868155153f..2c8142dbf03af 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 3e6784e8e4d01..1f7d153b7d99c 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index 43bbf934c74f8..f1ee778d96e7d 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 1c8c8df24f8b3..320b49f697bc6 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index 9c7ff64abd4e2..578868fe1b27e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index 82427ff3bfec1..ab10ba33daf65 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 0745982b12986..732b9c81c909f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index 694674f8cdeb0..4af6cc06140f2 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${fleet.system.index.version} }, "properties": { "agent": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index 81c95b063acd1..6b08dd183ed6d 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta" : { "version": "${xpack.ml.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index 0e48b8f427b43..764da07bd192e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -1,8 +1,8 @@ { "_doc" : { "_meta" : { - "version": "${xpack.ml.version", - "system-index-mappings-version": 1 + "version": "${xpack.ml.version}", + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index 3b5cf5f91dec9..a4e00cd244666 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta": { "version": "${xpack.ml.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index dc52d538c09d1..664469fc82c62 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta": { "version": "${xpack.ml.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index 42087ad363c84..9c6d50e49b03e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,7 +1,7 @@ { "_meta" : { "version": "${xpack.ml.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index cf82f52aac9ee..6a0d5bc8ccb21 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,7 +1,7 @@ { "_meta": { "version": "${xpack.ml.version}", - "system-index-mappings-version": 1 + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic": false, "properties" : { diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index ed36a998933ad..239d991d6d1c9 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -9,6 +9,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.ResourceNotFoundException; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; @@ -348,7 +349,12 @@ public void cleanUpFeature(ClusterService clusterService, Client client, ActionL } private String loadTemplateSource(String resource, int mappingsVersion) { - return TemplateUtils.loadTemplate(resource, Integer.toString(mappingsVersion), MAPPING_VERSION_VARIABLE); + return TemplateUtils.loadTemplate( + resource, + Version.CURRENT.toString(), + MAPPING_VERSION_VARIABLE, + Map.of("fleet.system.index.version", Integer.toString(mappingsVersion)) + ); } @Override From c1670e209f8905f3402c70267b30f846045053d0 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 14:56:58 -0400 Subject: [PATCH 19/34] Get the ML results index --- .../xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java | 5 ++++- .../ml/anomalydetection/results_index_mappings.json | 3 ++- .../java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 0dbed16d3b821..833f269721e6e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.core.template.TemplateUtils; import java.util.Locale; +import java.util.Map; import static org.elasticsearch.xpack.core.ClientHelper.ML_ORIGIN; import static org.elasticsearch.xpack.core.ClientHelper.executeAsyncWithOrigin; @@ -29,6 +30,7 @@ public final class AnomalyDetectorsIndex { private static final String RESULTS_MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; private static final String RESOURCE_PATH = "/ml/anomalydetection/"; + public static final int RESULTS_INDEX_MAPPINGS_VERSION = 1; private AnomalyDetectorsIndex() {} @@ -136,7 +138,8 @@ public static String resultsMapping() { return TemplateUtils.loadTemplate( RESOURCE_PATH + "results_index_mappings.json", Version.CURRENT.toString(), - RESULTS_MAPPINGS_VERSION_VARIABLE + RESULTS_MAPPINGS_VERSION_VARIABLE, + Map.of("xpack.ml.system.index.version", Integer.toString(RESULTS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json index 89389012664b2..b4de925629487 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json @@ -1,6 +1,7 @@ { "_meta" : { - "version" : "${xpack.ml.version}" + "version" : "${xpack.ml.version}", + "system-index-mappings-version": ${xpack.ml.system.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 3cf216d9f8de9..ac33ab3962a82 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -13,6 +13,7 @@ import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; +import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; import org.elasticsearch.xpack.test.rest.XPackRestTestHelper; @@ -133,7 +134,7 @@ private void assertUpgradedResultsMappings() throws Exception { assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( - InferenceIndexConstants.INFERENCE_INDEX_MAPPINGS_VERSION, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION, extractValue("mappings._meta.system-index-mappings-version", indexLevel) ); From 7c7760064ef89bf2c0c81db9baaaaebc3f62b6ae Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 14:59:30 -0400 Subject: [PATCH 20/34] Drop version numbers to 1 --- .../java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java | 2 +- .../xpack/core/ml/annotations/AnnotationIndex.java | 2 +- .../core/ml/inference/persistence/InferenceIndexConstants.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 62af62cd74ff9..17cef24bd13ac 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -20,7 +20,7 @@ public final class MlConfigIndex { private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; public static final int CONFIG_INDEX_MAX_RESULTS_WINDOW = 10_000; - public static final int CONFIG_INDEX_MAPPINGS_VERSION = 2; + public static final int CONFIG_INDEX_MAPPINGS_VERSION = 1; /** * The name of the index where job, datafeed and analytics configuration is stored diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 79949f5d4b141..5b420e4f9c994 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -57,7 +57,7 @@ public class AnnotationIndex { public static final List OLD_INDEX_NAMES = List.of(".ml-annotations-6"); private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - public static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 3; + public static final int ANNOTATION_INDEX_MAPPINGS_VERSION = 1; /** * Create the .ml-annotations-6 index with correct mappings if it does not already exist. This index is read and written by the UI diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index 3f9e3b515ee4b..d4bb3815a8dd9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -46,7 +46,7 @@ public final class InferenceIndexConstants { private static final String NATIVE_LATEST_INDEX = NATIVE_INDEX_PREFIX + NATIVE_INDEX_VERSION; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - public static final int INFERENCE_INDEX_MAPPINGS_VERSION = 4; + public static final int INFERENCE_INDEX_MAPPINGS_VERSION = 1; public static String mapping() { return TemplateUtils.loadTemplate( From 43e42830c7f456354950c7b3c64ea4eb404f229b Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 15:31:21 -0400 Subject: [PATCH 21/34] spotless --- .../java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index ac33ab3962a82..250a3b0025b9f 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -12,7 +12,6 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.xpack.core.ml.MlConfigIndex; import org.elasticsearch.xpack.core.ml.annotations.AnnotationIndex; -import org.elasticsearch.xpack.core.ml.inference.persistence.InferenceIndexConstants; import org.elasticsearch.xpack.core.ml.job.persistence.AnomalyDetectorsIndex; import org.elasticsearch.xpack.test.rest.IndexMappingTemplateAsserter; import org.elasticsearch.xpack.test.rest.XPackRestTestConstants; From 3fe560dce7a72cd1abc84d51a7fa1c304ddb17d3 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 17:00:04 -0400 Subject: [PATCH 22/34] snake case is more idiomatic for json --- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 2 +- .../src/main/resources/fleet-actions-results.json | 2 +- .../src/main/resources/fleet-actions.json | 2 +- .../template-resources/src/main/resources/fleet-agents.json | 2 +- .../src/main/resources/fleet-artifacts.json | 2 +- .../src/main/resources/fleet-enrollment-api-keys.json | 2 +- .../src/main/resources/fleet-policies-leader.json | 2 +- .../src/main/resources/fleet-policies.json | 2 +- .../src/main/resources/fleet-secrets.json | 2 +- .../src/main/resources/fleet-servers.json | 2 +- .../src/main/resources/ml/annotations_index_mappings.json | 2 +- .../ml/anomalydetection/results_index_mappings.json | 2 +- .../src/main/resources/ml/config_index_mappings.json | 2 +- .../src/main/resources/ml/inference_index_mappings.json | 2 +- .../src/main/resources/ml/meta_index_mappings.json | 2 +- .../src/main/resources/ml/notifications_index_mappings.json | 2 +- .../src/main/resources/ml/stats_index_mappings.json | 2 +- .../org/elasticsearch/upgrades/MlMappingsUpgradeIT.java | 6 +++--- 18 files changed, 20 insertions(+), 20 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 93bc88d514768..96f297b46b533 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -138,7 +138,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable Date: Thu, 27 Jul 2023 17:44:46 -0400 Subject: [PATCH 23/34] Don't require correct order in CreateSystemIndicesIT --- .../admin/indices/create/CreateSystemIndicesIT.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index f36c2aae7f8ad..61743ec858656 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -29,6 +29,7 @@ import org.elasticsearch.cluster.metadata.Template; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.indices.TestSystemIndexDescriptor; import org.elasticsearch.indices.TestSystemIndexDescriptorAllowsTemplates; import org.elasticsearch.indices.TestSystemIndexPlugin; @@ -352,7 +353,6 @@ private void assertHasAliases(Set aliasNames, String name, String primar /** * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. - * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters. */ private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) { final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) @@ -366,11 +366,7 @@ private void assertMappingsAndSettings(String expectedMappings, String concreteI ); final Map sourceAsMap = mappings.get(concreteIndex).getSourceAsMap(); - try { - assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false))); final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME)) .actionGet(); From ea8d8bd7202edd5c5bdea68a5c9535968eba90c8 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 17:45:31 -0400 Subject: [PATCH 24/34] spotless --- .../action/admin/indices/create/CreateSystemIndicesIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java index 61743ec858656..f5a2121b2dde9 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java @@ -39,8 +39,6 @@ import org.junit.After; import org.junit.Before; -import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -52,7 +50,6 @@ import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; -import static org.elasticsearch.test.XContentTestUtils.convertToXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; From bede58bc796d62e414ef8b79003b88526cfd25f5 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 27 Jul 2023 18:27:03 -0400 Subject: [PATCH 25/34] Don't require correct order in mapping update IT either --- .../indices/SystemIndexMappingUpdateServiceIT.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java index 279f56f33c1a7..486aa8d1f5084 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java @@ -17,13 +17,12 @@ import org.elasticsearch.cluster.metadata.MappingMetadata; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.test.ESIntegTestCase; import org.elasticsearch.xcontent.XContentType; import org.junit.Before; -import java.io.IOException; -import java.io.UncheckedIOException; import java.util.Collection; import java.util.List; import java.util.Locale; @@ -31,7 +30,6 @@ import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME; import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME; -import static org.elasticsearch.test.XContentTestUtils.convertToXContent; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.hamcrest.Matchers.equalTo; @@ -122,7 +120,6 @@ private void triggerClusterStateUpdates() { /** * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values. - * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters. */ private void assertMappingsAndSettings(String expectedMappings) { final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME)) @@ -136,11 +133,7 @@ private void assertMappingsAndSettings(String expectedMappings) { ); final Map sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap(); - try { - assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings)); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false))); final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME)) .actionGet(); From a57809be19043bdf8897ec748bc2d0ab26ab1d86 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 28 Jul 2023 12:05:51 -0400 Subject: [PATCH 26/34] Add unit tests --- .../indices/SystemIndexDescriptor.java | 10 ++- .../indices/SystemIndexDescriptorTests.java | 75 ++++++++++++++++++- 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index 96f297b46b533..b1a656d4b30c8 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -281,7 +281,8 @@ protected SystemIndexDescriptor( // TODO[wrb]: extract version int and non-metadata hash something different this.oldMappingVersion = extractLegacyVersionFromMappings(mappings, versionMetaKey); this.mappingVersion = extractVersionFromMappings(mappings); - assert mappingVersion.version >= 0; + assert mappingVersion.version >= 0 : "The mappings version must not be negative"; + } else { assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings"; assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings"; @@ -529,6 +530,13 @@ public Version getOldMappingVersion() { return oldMappingVersion; } + public MappingVersion getMappingsVersion() { + if (isAutomaticallyManaged() == false) { + throw new IllegalStateException(this + " is not managed so there are no mappings or version"); + } + return mappingVersion; + } + /** * Gets a standardized message when the node contains a data or master node whose version is less * than that of the minimum supported version of this descriptor and its prior descriptors. diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 63da869665ee0..0b2f0e98d4536 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -25,10 +25,10 @@ import static org.elasticsearch.indices.SystemIndexDescriptor.findDynamicMapping; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; public class SystemIndexDescriptorTests extends ESTestCase { - // private static final String MAPPINGS = "{ \"_doc\": { \"_meta\": { \"version\": \"7.4.0\" } } }"; private static final String MAPPINGS = String.format(Locale.ROOT, """ { "_doc": { @@ -353,6 +353,79 @@ public void testManagedSystemIndexMustHaveMatchingIndexFormat() { assertThat(e.getMessage(), equalTo("Descriptor index format does not match index format in managed settings")); } + public void testUnmanagedIndexMappingsVersion() { + SystemIndexDescriptor indexDescriptor = SystemIndexDescriptor.builder() + .setIndexPattern(".unmanaged-*") + .setDescription("an unmanaged system index") + .setType(Type.INTERNAL_UNMANAGED) + .build(); + + IllegalStateException e = expectThrows(IllegalStateException.class, indexDescriptor::getMappingsVersion); + + assertThat(e.getMessage(), containsString("is not managed so there are no mappings or version")); + } + + // test mapping versions can't be negative + public void testNegativeMappingsVersion() { + int negativeVersion = randomIntBetween(Integer.MIN_VALUE, -1); + String mappings = String.format(Locale.ROOT, """ + { + "_doc": { + "_meta": { + "version": "7.4.0", + "%s": %d + } + } + } + """, SystemIndexDescriptor.VERSION_META_KEY, negativeVersion); + + SystemIndexDescriptor.Builder builder = priorSystemIndexDescriptorBuilder().setMappings(mappings); + + AssertionError e = expectThrows(AssertionError.class, builder::build); + + assertThat(e.getMessage(), equalTo("The mappings version must not be negative")); + } + + public void testHashesIgnoreMappingMetadata() { + String mappingFormatString = """ + { + "_doc": { + "_meta": { + "version": "%s", + "%s": %d + } + }, + "properties": { + "age": { "type": "integer" }, + "email": { "type": "keyword" }, + "name": { "type": "text" } + } + } + """; + + String mappings1 = String.format( + Locale.ROOT, + mappingFormatString, + "8.9.0", + SystemIndexDescriptor.VERSION_META_KEY, + randomIntBetween(1, 10) + ); + String mappings2 = String.format( + Locale.ROOT, + mappingFormatString, + "8.10.0", + SystemIndexDescriptor.VERSION_META_KEY, + randomIntBetween(11, 20) + ); + + SystemIndexDescriptor descriptor1 = priorSystemIndexDescriptorBuilder().setMappings(mappings1).build(); + SystemIndexDescriptor descriptor2 = priorSystemIndexDescriptorBuilder().setMappings(mappings2).build(); + + assertThat(descriptor1.getMappingsVersion().hash(), equalTo(descriptor2.getMappingsVersion().hash())); + assertThat(descriptor1.getMappingsVersion().version(), not(equalTo(descriptor2.getMappingsVersion().version()))); + assertThat(descriptor1.getOldMappingVersion(), not(equalTo(descriptor2.getOldMappingVersion()))); + } + private SystemIndexDescriptor.Builder priorSystemIndexDescriptorBuilder() { return SystemIndexDescriptor.builder() .setIndexPattern(".system*") From 1d2be998d2f09dd2894113dfa83c835546ca044d Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 28 Jul 2023 13:37:56 -0400 Subject: [PATCH 27/34] spotless --- .../org/elasticsearch/indices/SystemIndexDescriptorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 0b2f0e98d4536..33d1822cd95d4 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -402,7 +402,7 @@ public void testHashesIgnoreMappingMetadata() { } } """; - + String mappings1 = String.format( Locale.ROOT, mappingFormatString, From 8108e790f03b139d10cb1938954f627e19c7bbb0 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Fri, 28 Jul 2023 15:36:01 -0400 Subject: [PATCH 28/34] Clean up naming of deprecated Version --- .../indices/SystemIndexDescriptor.java | 47 ++++++++++--------- .../SystemIndexMappingUpdateService.java | 4 +- .../indices/SystemIndexDescriptorTests.java | 2 +- .../support/SecurityIndexManager.java | 2 +- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index b1a656d4b30c8..df168ed8de380 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -132,10 +132,10 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable_meta in the index mappings - * that contains the index's mappings' version. + * that contains the index's mappings' {@link Version}. We need to read and write this field for + * backwards compatibility. */ - // TODO[wrb]: need to keep this for bwc & finding the field to update, but we can standardize a key under _meta - private final String legacyVersionMetaKey; + private final String mappingsNodeVersionMetaKey; /** The version meta key for the integer system index mapping version */ public static final String VERSION_META_KEY = "system_index_mappings_version"; @@ -149,10 +149,10 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable_meta where a version can be found, which indicates the + * @param mappingsNodeVersionMetaKey a mapping key under _meta where a version can be found, which indicates the * Elasticsearch version when the index was created. * @param origin the client origin to use when creating this index. Internal system indices must not provide an origin, while external * system indices must do so. @@ -220,7 +220,7 @@ protected SystemIndexDescriptor( Settings settings, String aliasName, int indexFormat, - String versionMetaKey, + String mappingsNodeVersionMetaKey, String origin, Version minimumNodeVersion, Type type, @@ -273,23 +273,23 @@ protected SystemIndexDescriptor( Objects.requireNonNull(settings, "Must supply settings for a managed system index"); Strings.requireNonEmpty(mappings, "Must supply mappings for a managed system index"); Strings.requireNonEmpty(primaryIndex, "Must supply primaryIndex for a managed system index"); - Strings.requireNonEmpty(versionMetaKey, "Must supply versionMetaKey for a managed system index"); + Strings.requireNonEmpty(mappingsNodeVersionMetaKey, "Must supply nodeVersionMetaKey for a managed system index"); Strings.requireNonEmpty(origin, "Must supply origin for a managed system index"); if (settings.getAsInt(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 0) != indexFormat) { throw new IllegalArgumentException("Descriptor index format does not match index format in managed settings"); } // TODO[wrb]: extract version int and non-metadata hash something different - this.oldMappingVersion = extractLegacyVersionFromMappings(mappings, versionMetaKey); - this.mappingVersion = extractVersionFromMappings(mappings); - assert mappingVersion.version >= 0 : "The mappings version must not be negative"; + this.mappingsNodeVersion = extractNodeVersionFromMappings(mappings, mappingsNodeVersionMetaKey); + this.mappingsVersion = extractVersionFromMappings(mappings); + assert mappingsVersion.version >= 0 : "The mappings version must not be negative"; } else { assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings"; assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings"; assert Objects.isNull(primaryIndex) : "Unmanaged index descriptors should not have a primary index"; - assert Objects.isNull(versionMetaKey) : "Unmanaged index descriptors should not have a version meta key"; - this.oldMappingVersion = null; - this.mappingVersion = null; + assert Objects.isNull(mappingsNodeVersionMetaKey) : "Unmanaged index descriptors should not have a version meta key"; + this.mappingsNodeVersion = null; + this.mappingsVersion = null; } Objects.requireNonNull(allowedElasticProductOrigins, "allowedProductOrigins must not be null"); @@ -372,7 +372,7 @@ protected SystemIndexDescriptor( throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true."); } this.indexFormat = indexFormat; - this.legacyVersionMetaKey = versionMetaKey; + this.mappingsNodeVersionMetaKey = mappingsNodeVersionMetaKey; this.origin = origin; this.minimumNodeVersion = minimumNodeVersion; this.type = type; @@ -467,9 +467,9 @@ public int getIndexFormat() { return this.indexFormat; } - public String getLegacyVersionMetaKey() { + public String getMappingsNodeVersionMetaKey() { assert isAutomaticallyManaged() : "Do not request version meta keys for unmanaged system indices"; - return this.legacyVersionMetaKey; + return this.mappingsNodeVersionMetaKey; } public Version getMinimumNodeVersion() { @@ -523,18 +523,23 @@ public boolean allowsTemplates() { return allowsTemplates; } - public Version getOldMappingVersion() { + /** + * Use of the mappings {@link Version} should be replaced with the value returned from {@link #getMappingsVersion()} + * @return Elasticsearch version associated with this descriptor's mappings. + */ + @Deprecated + public Version getMappingsNodeVersion() { if (isAutomaticallyManaged() == false) { throw new IllegalStateException(this + " is not managed so there are no mappings or version"); } - return oldMappingVersion; + return mappingsNodeVersion; } public MappingVersion getMappingsVersion() { if (isAutomaticallyManaged() == false) { throw new IllegalStateException(this + " is not managed so there are no mappings or version"); } - return mappingVersion; + return mappingsVersion; } /** @@ -882,7 +887,7 @@ private static MappingVersion extractVersionFromMappings(String mappings) { // TODO[wrb]: remove method @SuppressWarnings("unchecked") - private static Version extractLegacyVersionFromMappings(String mappings, String versionMetaKey) { + private static Version extractNodeVersionFromMappings(String mappings, String versionMetaKey) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false); final Map doc = (Map) mappingsMap.get("_doc"); final Map meta; diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index b078d308db344..bacd25210cd3c 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -292,7 +292,7 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp return Version.V_EMPTY; } - final Object rawVersion = meta.get(descriptor.getLegacyVersionMetaKey()); + final Object rawVersion = meta.get(descriptor.getMappingsNodeVersionMetaKey()); if (rawVersion instanceof Integer) { // This can happen with old system indices, such as .tasks, which were created before we used an Elasticsearch // version here. We should just replace the template to be sure. @@ -302,7 +302,7 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp if (versionString == null) { logger.warn( "No value found in mappings for [_meta.{}], assuming mappings update required", - descriptor.getLegacyVersionMetaKey() + descriptor.getMappingsNodeVersionMetaKey() ); // If we called `Version.fromString(null)`, it would return `Version.CURRENT` and we wouldn't update the mappings return Version.V_EMPTY; diff --git a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java index 33d1822cd95d4..7a651dea0c878 100644 --- a/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java +++ b/server/src/test/java/org/elasticsearch/indices/SystemIndexDescriptorTests.java @@ -423,7 +423,7 @@ public void testHashesIgnoreMappingMetadata() { assertThat(descriptor1.getMappingsVersion().hash(), equalTo(descriptor2.getMappingsVersion().hash())); assertThat(descriptor1.getMappingsVersion().version(), not(equalTo(descriptor2.getMappingsVersion().version()))); - assertThat(descriptor1.getOldMappingVersion(), not(equalTo(descriptor2.getOldMappingVersion()))); + assertThat(descriptor1.getMappingsNodeVersion(), not(equalTo(descriptor2.getMappingsNodeVersion()))); } private SystemIndexDescriptor.Builder priorSystemIndexDescriptorBuilder() { diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java index edae07a94a902..bdc25098f1760 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityIndexManager.java @@ -264,7 +264,7 @@ private boolean checkIndexMappingUpToDate(ClusterState clusterState) { * * ...which is true if the mappings have been updated. */ - return checkIndexMappingVersionMatches(clusterState, descriptor.getOldMappingVersion()::onOrBefore); + return checkIndexMappingVersionMatches(clusterState, descriptor.getMappingsNodeVersion()::onOrBefore); } private boolean checkIndexMappingVersionMatches(ClusterState clusterState, Predicate predicate) { From d1bc552ba598afe475f476e5019a3848227c9e90 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Thu, 3 Aug 2023 10:13:34 -0400 Subject: [PATCH 29/34] Rename version meta key ('system' -> 'managed') --- .../elasticsearch/indices/SystemIndexDescriptor.java | 12 ++++++------ .../elasticsearch/xpack/core/ml/MlConfigIndex.java | 2 +- .../org/elasticsearch/xpack/core/ml/MlMetaIndex.java | 2 +- .../elasticsearch/xpack/core/ml/MlStatsIndex.java | 2 +- .../xpack/core/ml/annotations/AnnotationIndex.java | 2 +- .../persistence/InferenceIndexConstants.java | 2 +- .../ml/job/persistence/AnomalyDetectorsIndex.java | 2 +- .../core/ml/notifications/NotificationsIndex.java | 2 +- .../src/main/resources/fleet-actions-results.json | 2 +- .../src/main/resources/fleet-actions.json | 2 +- .../src/main/resources/fleet-agents.json | 2 +- .../src/main/resources/fleet-artifacts.json | 2 +- .../main/resources/fleet-enrollment-api-keys.json | 2 +- .../src/main/resources/fleet-policies-leader.json | 2 +- .../src/main/resources/fleet-policies.json | 2 +- .../src/main/resources/fleet-secrets.json | 2 +- .../src/main/resources/fleet-servers.json | 2 +- .../resources/ml/annotations_index_mappings.json | 2 +- .../ml/anomalydetection/results_index_mappings.json | 2 +- .../src/main/resources/ml/config_index_mappings.json | 2 +- .../main/resources/ml/inference_index_mappings.json | 2 +- .../src/main/resources/ml/meta_index_mappings.json | 2 +- .../resources/ml/notifications_index_mappings.json | 2 +- .../src/main/resources/ml/stats_index_mappings.json | 2 +- .../java/org/elasticsearch/xpack/fleet/Fleet.java | 2 +- .../elasticsearch/upgrades/MlMappingsUpgradeIT.java | 6 +++--- 26 files changed, 33 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index df168ed8de380..e58af4312ac8d 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -138,7 +138,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable map) { } @SuppressWarnings("unchecked") // we do a lot of casting of maps - private static MappingVersion extractVersionFromMappings(String mappings) { + private static MappingsVersion extractVersionFromMappings(String mappings) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, true); final Map doc = (Map) mappingsMap.get("_doc"); final Map meta; @@ -882,7 +882,7 @@ private static MappingVersion extractVersionFromMappings(String mappings) { if (value == null) { throw new IllegalArgumentException("mappings do not have a version in _meta." + VERSION_META_KEY); } - return new MappingVersion(value, Objects.hash(properties)); + return new MappingsVersion(value, Objects.hash(properties)); } // TODO[wrb]: remove method diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java index 17cef24bd13ac..da905ef5efdee 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlConfigIndex.java @@ -36,7 +36,7 @@ public static String mapping() { "/ml/config_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(CONFIG_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java index 04c4b26bc9e95..9190be89ee1e0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlMetaIndex.java @@ -34,7 +34,7 @@ public static String mapping() { "/ml/meta_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(META_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(META_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java index 10549f42f5a8a..f7c6702a3ec6c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java @@ -42,7 +42,7 @@ public static String mapping() { "/ml/stats_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(STATS_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(STATS_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 5b420e4f9c994..3310e04c37fa0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -214,7 +214,7 @@ public static String annotationsMapping() { "/ml/annotations_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(ANNOTATION_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java index d4bb3815a8dd9..1e60da895c0e7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/inference/persistence/InferenceIndexConstants.java @@ -53,7 +53,7 @@ public static String mapping() { "/ml/inference_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(INFERENCE_INDEX_MAPPINGS_VERSION)) ); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java index 833f269721e6e..29ac3359c870b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/AnomalyDetectorsIndex.java @@ -139,7 +139,7 @@ public static String resultsMapping() { RESOURCE_PATH + "results_index_mappings.json", Version.CURRENT.toString(), RESULTS_MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(RESULTS_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(RESULTS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java index 18ecf1379c10a..06a923cd9d275 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/notifications/NotificationsIndex.java @@ -26,7 +26,7 @@ public static String mapping() { RESOURCE_PATH + "notifications_index_mappings.json", Version.CURRENT.toString(), MAPPINGS_VERSION_VARIABLE, - Map.of("xpack.ml.system.index.version", Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION)) + Map.of("xpack.ml.managed.index.version", Integer.toString(NOTIFICATIONS_INDEX_MAPPINGS_VERSION)) ); } } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json index 2546c276a6dd9..88c51a9aef284 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions-results.json @@ -11,7 +11,7 @@ "mappings": { "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json index d1bacbd24701a..2b3ecbac92352 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-actions.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "action_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json index 5fe1755874c74..6b96b204b4f7f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-agents.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "access_api_key_id": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json index 920ed12585bb1..4a5f76470d24f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-artifacts.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "identifier": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json index 0a953f352375a..6be455e02825a 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-enrollment-api-keys.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "active": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json index 43b2b862a8f5e..7cce4e0d4f53a 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies-leader.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "server": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json index f58e1c5a1371f..44e2e67dd06c3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-policies.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "coordinator_idx": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json index 475ccb5adb7eb..6d16098e38049 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-secrets.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "value": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json index 92bd6315af865..8ddd5e7e0693e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/fleet-servers.json @@ -7,7 +7,7 @@ "dynamic": false, "_meta": { "version": "${fleet.version}", - "system_index_mappings_version": ${fleet.system.index.version} + "managed_index_mappings_version": ${fleet.managed.index.version} }, "properties": { "agent": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json index 245ac142523fb..42efa741dd99d 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/annotations_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta" : { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic" : false, "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json index 10f8c78603a36..4415afe50a998 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/anomalydetection/results_index_mappings.json @@ -1,7 +1,7 @@ { "_meta" : { "version" : "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json index 146461bfc397b..811810f5d0f2f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/config_index_mappings.json @@ -2,7 +2,7 @@ "_doc" : { "_meta" : { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates" : [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json index 22705ed62ec60..7ff961a0aac9c 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/inference_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta": { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic": false, "properties": { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json index 86c1d7adaa647..4606cb0d75d8f 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/meta_index_mappings.json @@ -2,7 +2,7 @@ "_doc": { "_meta": { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic_templates": [ { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json index cd273d45309b0..b2135d2c6e76e 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/notifications_index_mappings.json @@ -1,7 +1,7 @@ { "_meta" : { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic" : "false", "properties" : { diff --git a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json index bf088c89ab542..8c151cb38c1b3 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/ml/stats_index_mappings.json @@ -1,7 +1,7 @@ { "_meta": { "version": "${xpack.ml.version}", - "system_index_mappings_version": ${xpack.ml.system.index.version} + "managed_index_mappings_version": ${xpack.ml.managed.index.version} }, "dynamic": false, "properties" : { diff --git a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java index a60a4f2747984..009e17bd79e6b 100644 --- a/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java +++ b/x-pack/plugin/fleet/src/main/java/org/elasticsearch/xpack/fleet/Fleet.java @@ -362,7 +362,7 @@ private String loadTemplateSource(String resource, int mappingsVersion) { resource, Version.CURRENT.toString(), MAPPING_VERSION_VARIABLE, - Map.of("fleet.system.index.version", Integer.toString(mappingsVersion)) + Map.of("fleet.managed.index.version", Integer.toString(mappingsVersion)) ); } diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java index 92f7cebfc92dd..95d543a0f63ea 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/MlMappingsUpgradeIT.java @@ -134,7 +134,7 @@ private void assertUpgradedResultsMappings() throws Exception { assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION, - extractValue("mappings._meta.system_index_mappings_version", indexLevel) + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) ); // TODO: as the years go by, the field we assert on here should be changed @@ -171,7 +171,7 @@ private void assertUpgradedAnnotationsMappings() throws Exception { assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( AnnotationIndex.ANNOTATION_INDEX_MAPPINGS_VERSION, - extractValue("mappings._meta.system_index_mappings_version", indexLevel) + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) ); // TODO: as the years go by, the field we assert on here should be changed @@ -229,7 +229,7 @@ private void assertUpgradedConfigMappings() throws Exception { assertEquals(Version.CURRENT.toString(), extractValue("mappings._meta.version", indexLevel)); assertEquals( MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION, - extractValue("mappings._meta.system_index_mappings_version", indexLevel) + extractValue("mappings._meta.managed_index_mappings_version", indexLevel) ); // TODO: as the years go by, the field we assert on here should be changed From 03c2a7ce2084a14c2d022640b2adcdea620cc268 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Thu, 3 Aug 2023 17:00:00 +0100 Subject: [PATCH 30/34] Update mappings for ML indices if required --- .../persistence/ElasticsearchMappings.java | 8 +++ .../ElasticsearchMappingsTests.java | 67 +++++++++++++++---- 2 files changed, 63 insertions(+), 12 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 1bb88f4bc4773..0730ce7b723e9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.CheckedSupplier; import org.elasticsearch.core.TimeValue; import org.elasticsearch.index.Index; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transports; @@ -113,6 +114,13 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic @SuppressWarnings("unchecked") Map meta = (Map) metadata.sourceAsMap().get("_meta"); if (meta != null) { + Integer systemIndexMappingsVersion = (Integer) meta.get(SystemIndexDescriptor.VERSION_META_KEY); + if (systemIndexMappingsVersion == null) { + logger.info("System index mappings version for [{}] not found, recreating", index); + indicesToUpdate.add(index); + continue; + } + String versionString = (String) meta.get("version"); if (versionString == null) { logger.info("Version of mappings for [{}] not found, recreating", index); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index cf9708505d075..097a967585203 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.get.GetResult; +import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.VersionUtils; @@ -146,9 +147,21 @@ public void testMappingRequiresUpdateNoVersion() { } public void testMappingRequiresUpdateRecentMappingVersion() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_current", Version.CURRENT.toString())); - String[] indices = new String[] { "version_current" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + } } public void testMappingRequiresUpdateMaliciousMappingVersion() { @@ -166,18 +179,41 @@ public void testMappingRequiresUpdateBogusMappingVersion() { } public void testMappingRequiresUpdateNewerMappingVersion() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT)); - String[] indices = new String[] { "version_newer" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT)); + String[] indices = new String[] { "version_newer" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[]{"version_newer"}; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + } } public void testMappingRequiresUpdateNewerMappingVersionMinor() { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); - String[] indices = new String[] { "version_newer_minor" }; - assertArrayEquals( - new String[] {}, - ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) - ); + { + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); + String[] indices = new String[]{"version_newer_minor"}; + assertArrayEquals( + indices, + ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) + ); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer_minor", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_newer_minor" }; + assertArrayEquals( + new String[] {}, + ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) + ); + } } @SuppressWarnings({ "unchecked" }) @@ -215,6 +251,10 @@ public void testAddDocMappingIfMissing() { } private ClusterState getClusterStateWithMappingsWithMetadata(Map namesAndVersions) { + return getClusterStateWithMappingsWithMetadata(namesAndVersions, null); + } + + private ClusterState getClusterStateWithMappingsWithMetadata(Map namesAndVersions, Map metaData) { Metadata.Builder metadataBuilder = Metadata.builder(); for (Map.Entry entry : namesAndVersions.entrySet()) { @@ -236,6 +276,9 @@ private ClusterState getClusterStateWithMappingsWithMetadata(Map if (version != null && version.equals("NO_VERSION_FIELD") == false) { meta.put("version", version); } + if (metaData != null) { + metaData.forEach((k, v) -> meta.putIfAbsent(k, v)); + } mapping.put("_meta", meta); indexMetadata.putMapping(new MappingMetadata("_doc", mapping)); From 9f2eae4a6bc14e0603ca7723247f92d773731d42 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 4 Aug 2023 09:07:16 +0100 Subject: [PATCH 31/34] Formatting --- .../ml/job/persistence/ElasticsearchMappingsTests.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index 097a967585203..ac7fe3c055c8c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -189,7 +189,7 @@ public void testMappingRequiresUpdateNewerMappingVersion() { Collections.singletonMap("version_newer", Version.CURRENT), Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) ); - String[] indices = new String[]{"version_newer"}; + String[] indices = new String[] { "version_newer" }; assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); } } @@ -197,11 +197,8 @@ public void testMappingRequiresUpdateNewerMappingVersion() { public void testMappingRequiresUpdateNewerMappingVersionMinor() { { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); - String[] indices = new String[]{"version_newer_minor"}; - assertArrayEquals( - indices, - ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) - ); + String[] indices = new String[] { "version_newer_minor" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion())); } { ClusterState cs = getClusterStateWithMappingsWithMetadata( From fced7270ce3184f3ab1a361cd49953b0624c877c Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 4 Aug 2023 11:55:51 +0100 Subject: [PATCH 32/34] Trigger ML index mappings updates based on new index mappings version. --- .../xpack/core/ml/MlStatsIndex.java | 2 +- .../core/ml/annotations/AnnotationIndex.java | 3 +- .../persistence/ElasticsearchMappings.java | 27 +++++----- .../ElasticsearchMappingsTests.java | 49 ++++++++++++------- .../ml/action/TransportOpenJobAction.java | 3 +- .../TransportPutDataFrameAnalyticsAction.java | 3 +- .../TransportRevertModelSnapshotAction.java | 3 +- ...ansportUpdateDataFrameAnalyticsAction.java | 3 +- ...ransportUpgradeJobModelSnapshotAction.java | 3 +- .../xpack/ml/datafeed/DatafeedManager.java | 6 ++- .../dataframe/DataFrameAnalyticsManager.java | 3 +- .../inference/TrainedModelStatsService.java | 3 +- .../xpack/ml/job/JobManager.java | 6 ++- .../autodetect/AutodetectProcessManager.java | 6 ++- .../upgrader/SnapshotUpgradeTaskExecutor.java | 6 ++- .../task/OpenJobPersistentTasksExecutor.java | 3 +- 16 files changed, 78 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java index f7c6702a3ec6c..83c7a3fe45a09 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/MlStatsIndex.java @@ -26,7 +26,7 @@ public class MlStatsIndex { public static final String TEMPLATE_NAME = ".ml-stats"; private static final String MAPPINGS_VERSION_VARIABLE = "xpack.ml.version"; - private static final int STATS_INDEX_MAPPINGS_VERSION = 1; + public static final int STATS_INDEX_MAPPINGS_VERSION = 1; private MlStatsIndex() {} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java index 3310e04c37fa0..745e50c920288 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/annotations/AnnotationIndex.java @@ -104,7 +104,8 @@ public static void createAnnotationsIndexIfNecessary( client, state, masterNodeTimeout, - finalListener + finalListener, + ANNOTATION_INDEX_MAPPINGS_VERSION ), finalListener::onFailure ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java index 0730ce7b723e9..1dde9dc6075d0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappings.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.elasticsearch.ElasticsearchException; -import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.admin.indices.mapping.put.PutMappingAction; @@ -101,7 +100,7 @@ public class ElasticsearchMappings { private ElasticsearchMappings() {} - static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, Version minVersion) { + static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndices, int minVersion) { List indicesToUpdate = new ArrayList<>(); Map currentMapping = state.metadata() @@ -121,22 +120,19 @@ static String[] mappingRequiresUpdate(ClusterState state, String[] concreteIndic continue; } - String versionString = (String) meta.get("version"); - if (versionString == null) { - logger.info("Version of mappings for [{}] not found, recreating", index); - indicesToUpdate.add(index); - continue; - } - - Version mappingVersion = Version.fromString(versionString); - - if (mappingVersion.onOrAfter(minVersion)) { + if (systemIndexMappingsVersion >= minVersion) { continue; } else { - logger.info("Mappings for [{}] are outdated [{}], updating it[{}].", index, mappingVersion, Version.CURRENT); + logger.info( + "Mappings for [{}] are outdated [{}], updating it[{}].", + index, + systemIndexMappingsVersion, + minVersion + ); indicesToUpdate.add(index); continue; } + } else { logger.info("Version of mappings for [{}] not found, recreating", index); indicesToUpdate.add(index); @@ -161,7 +157,8 @@ public static void addDocMappingIfMissing( Client client, ClusterState state, TimeValue masterNodeTimeout, - ActionListener listener + ActionListener listener, + int minVersion ) { IndexAbstraction indexAbstraction = state.metadata().getIndicesLookup().get(alias); if (indexAbstraction == null) { @@ -175,7 +172,7 @@ public static void addDocMappingIfMissing( protected void doRun() throws Exception { String[] concreteIndices = indexAbstraction.getIndices().stream().map(Index::getName).toArray(String[]::new); - final String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, Version.CURRENT); + final String[] indicesThatRequireAnUpdate = mappingRequiresUpdate(state, concreteIndices, minVersion); if (indicesThatRequireAnUpdate.length > 0) { String mapping = mappingSupplier.get(); PutMappingRequest putMappingRequest = new PutMappingRequest(indicesThatRequireAnUpdate); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java index ac7fe3c055c8c..d95c94801e52a 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/persistence/ElasticsearchMappingsTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.indices.SystemIndexDescriptor; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.test.VersionUtils; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xcontent.XContentParseException; import org.elasticsearch.xcontent.XContentParser; @@ -131,19 +130,19 @@ public void testMappingRequiresUpdateNoMapping() { ClusterState cs = csBuilder.build(); String[] indices = new String[] { "no_index" }; - assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(new String[] { "no_index" }, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNullMapping() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("null_mapping", null)); String[] indices = new String[] { "null_index" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNoVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("no_version_field", "NO_VERSION_FIELD")); String[] indices = new String[] { "no_version_field" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateRecentMappingVersion() { @@ -152,7 +151,7 @@ public void testMappingRequiresUpdateRecentMappingVersion() { Collections.singletonMap("version_current", Version.CURRENT.toString()) ); String[] indices = new String[] { "version_current" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } { ClusterState cs = getClusterStateWithMappingsWithMetadata( @@ -160,7 +159,15 @@ public void testMappingRequiresUpdateRecentMappingVersion() { Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) ); String[] indices = new String[] { "version_current" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_current", Version.CURRENT.toString()), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) + ); + String[] indices = new String[] { "version_current" }; + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2)); } } @@ -169,20 +176,28 @@ public void testMappingRequiresUpdateMaliciousMappingVersion() { Collections.singletonMap("version_current", Collections.singletonMap("nested", "1.0")) ); String[] indices = new String[] { "version_nested" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateBogusMappingVersion() { ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_bogus", "0.0")); String[] indices = new String[] { "version_bogus" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, Version.CURRENT)); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } public void testMappingRequiresUpdateNewerMappingVersion() { { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", Version.CURRENT)); + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer", 2)); String[] indices = new String[] { "version_newer" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); + } + { + ClusterState cs = getClusterStateWithMappingsWithMetadata( + Collections.singletonMap("version_newer", Version.CURRENT), + Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 2) + ); + String[] indices = new String[] { "version_newer" }; + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } { ClusterState cs = getClusterStateWithMappingsWithMetadata( @@ -190,15 +205,15 @@ public void testMappingRequiresUpdateNewerMappingVersion() { Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) ); String[] indices = new String[] { "version_newer" }; - assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousVersion())); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 2)); } } public void testMappingRequiresUpdateNewerMappingVersionMinor() { { - ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", Version.CURRENT)); + ClusterState cs = getClusterStateWithMappingsWithMetadata(Collections.singletonMap("version_newer_minor", 1)); String[] indices = new String[] { "version_newer_minor" }; - assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion())); + assertArrayEquals(indices, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } { ClusterState cs = getClusterStateWithMappingsWithMetadata( @@ -206,10 +221,7 @@ public void testMappingRequiresUpdateNewerMappingVersionMinor() { Collections.singletonMap(SystemIndexDescriptor.VERSION_META_KEY, 1) ); String[] indices = new String[] { "version_newer_minor" }; - assertArrayEquals( - new String[] {}, - ElasticsearchMappings.mappingRequiresUpdate(cs, indices, VersionUtils.getPreviousMinorVersion()) - ); + assertArrayEquals(new String[] {}, ElasticsearchMappings.mappingRequiresUpdate(cs, indices, 1)); } } @@ -233,7 +245,8 @@ public void testAddDocMappingIfMissing() { client, clusterState, MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT, - ActionTestUtils.assertNoFailureListener(Assert::assertTrue) + ActionTestUtils.assertNoFailureListener(Assert::assertTrue), + 1 ); ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(PutMappingRequest.class); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java index a458c3df86910..5e033278242ad 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportOpenJobAction.java @@ -313,7 +313,8 @@ private void clearJobFinishedTime( client, clusterState, masterNodeTimeout, - mappingsUpdatedListener + mappingsUpdatedListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java index a4982addd1d08..1c0279947aafb 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportPutDataFrameAnalyticsAction.java @@ -273,7 +273,8 @@ private void updateDocMappingAndPutConfig( client, clusterState, masterNodeTimeout, - ActionListener.wrap(unused -> configProvider.put(config, headers, masterNodeTimeout, auditingListener), listener::onFailure) + ActionListener.wrap(unused -> configProvider.put(config, headers, masterNodeTimeout, auditingListener), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java index ecfbec71ab285..b12d1ffd75287 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportRevertModelSnapshotAction.java @@ -169,7 +169,8 @@ protected void masterOperation( client, state, request.masterNodeTimeout(), - configMappingUpdateListener + configMappingUpdateListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java index c542213029289..921adf3924dde 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpdateDataFrameAnalyticsAction.java @@ -110,7 +110,8 @@ protected void masterOperation( client, state, request.masterNodeTimeout(), - ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java index e08d25c3048ba..ded19b605e475 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/action/TransportUpgradeJobModelSnapshotAction.java @@ -174,7 +174,8 @@ protected void masterOperation(Task task, Request request, ClusterState state, A client, state, request.masterNodeTimeout(), - configIndexMappingUpdaterListener + configIndexMappingUpdaterListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java index 7fcf7cf980246..ef8b589735585 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/DatafeedManager.java @@ -239,7 +239,8 @@ public void updateDatafeed( client, state, request.masterNodeTimeout(), - ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); } @@ -332,7 +333,8 @@ private void putDatafeed( client, clusterState, request.masterNodeTimeout(), - ActionListener.wrap(mappingsUpdated, listener::onFailure) + ActionListener.wrap(mappingsUpdated, listener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); }; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java index e699f071e2cfc..718b70fb8801b 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/dataframe/DataFrameAnalyticsManager.java @@ -165,7 +165,8 @@ private void createStatsIndexAndUpdateMappingsIfNecessary( clientToUse, clusterState, masterNodeTimeout, - listener + listener, + MlStatsIndex.STATS_INDEX_MAPPINGS_VERSION ), listener::onFailure ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java index 6376b21f07c06..b6c6f0865aad8 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/TrainedModelStatsService.java @@ -262,7 +262,8 @@ private void createStatsIndexIfNecessary() { client, clusterState, MasterNodeRequest.DEFAULT_MASTER_NODE_TIMEOUT, - listener + listener, + MlStatsIndex.STATS_INDEX_MAPPINGS_VERSION ), listener::onFailure ) diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java index aa07df14a10d4..1e99f62d5c97d 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/JobManager.java @@ -282,7 +282,8 @@ public void onFailure(Exception e) { client, state, request.masterNodeTimeout(), - putJobListener + putJobListener, + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ), putJobListener::onFailure ); @@ -353,7 +354,8 @@ public void updateJob(UpdateJobAction.Request request, ActionListener doUpdate.run(), actionListener::onFailure) + ActionListener.wrap(bool -> doUpdate.run(), actionListener::onFailure), + MlConfigIndex.CONFIG_INDEX_MAPPINGS_VERSION ); if (request.getJobUpdate().getGroups() != null && request.getJobUpdate().getGroups().isEmpty() == false) { diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java index 9170950778444..f09b3166964f5 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/process/autodetect/AutodetectProcessManager.java @@ -574,7 +574,8 @@ public void openJob( client, clusterState, masterNodeTimeout, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ), e -> { // Due to a bug in 7.9.0 it's possible that the annotations index already has incorrect mappings @@ -586,7 +587,8 @@ public void openJob( client, clusterState, masterNodeTimeout, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java index 8119deb7674f9..69b926876302a 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/snapshot/upgrader/SnapshotUpgradeTaskExecutor.java @@ -198,7 +198,8 @@ protected void nodeOperation(AllocatedPersistentTask task, SnapshotUpgradeTaskPa client, clusterState, MlTasks.PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ), e -> { // Due to a bug in 7.9.0 it's possible that the annotations index already has incorrect mappings @@ -210,7 +211,8 @@ protected void nodeOperation(AllocatedPersistentTask task, SnapshotUpgradeTaskPa client, clusterState, MlTasks.PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } ); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java index e9f185605e7ec..b7efe40a72403 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/job/task/OpenJobPersistentTasksExecutor.java @@ -276,7 +276,8 @@ protected void nodeOperation(AllocatedPersistentTask task, OpenJobAction.JobPara client, clusterState, PERSISTENT_TASK_MASTER_NODE_TIMEOUT, - resultsMappingUpdateHandler + resultsMappingUpdateHandler, + AnomalyDetectorsIndex.RESULTS_INDEX_MAPPINGS_VERSION ); } From 8b4e3718f4e5fc285ef75ee2af62744506fb6448 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 15 Aug 2023 14:50:56 -0400 Subject: [PATCH 33/34] Remove TODOs tracked elsewhere --- .../org/elasticsearch/indices/SystemIndexDescriptor.java | 7 ------- .../indices/SystemIndexMappingUpdateService.java | 3 --- 2 files changed, 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java index e58af4312ac8d..4d37cea8bcf98 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java @@ -144,11 +144,9 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable= 0 : "The mappings version must not be negative"; @@ -311,7 +308,6 @@ protected SystemIndexDescriptor( Set versions = Sets.newHashSetWithExpectedSize(priorSystemIndexDescriptors.size() + 1); versions.add(minimumNodeVersion); for (SystemIndexDescriptor prior : priorSystemIndexDescriptors) { - // TODO[wrb]: use mapping version instead of minimum node version if (versions.add(prior.minimumNodeVersion) == false) { throw new IllegalArgumentException(prior + " has the same minimum node version as another descriptor"); } @@ -549,7 +545,6 @@ public MappingsVersion getMappingsVersion() { * @param cause the action being attempted that triggered the check. Used in the error message. * @return the standardized error message */ - // TODO[wrb]: update for new "mapping version" world public String getMinimumNodeVersionMessage(String cause) { Objects.requireNonNull(cause); final Version actualMinimumVersion = priorSystemIndexDescriptors.isEmpty() @@ -885,7 +880,6 @@ private static MappingsVersion extractVersionFromMappings(String mappings) { return new MappingsVersion(value, Objects.hash(properties)); } - // TODO[wrb]: remove method @SuppressWarnings("unchecked") private static Version extractNodeVersionFromMappings(String mappings, String versionMetaKey) { final Map mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false); @@ -901,7 +895,6 @@ private static Version extractNodeVersionFromMappings(String mappings, String ve } final String value = (String) meta.get(versionMetaKey); if (value == null) { - // TODO[wrb]: Tolerate null throw new IllegalArgumentException("mappings do not have a version in _meta." + versionMetaKey); } return Version.fromString(value); diff --git a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java index bacd25210cd3c..d1f8acfccc0ac 100644 --- a/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java +++ b/server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java @@ -93,7 +93,6 @@ public void clusterChanged(ClusterChangedEvent event) { } // if we're in a mixed-version cluster, exit - // TODO[wrb]: compare cluster minimum mapping version for system index descriptors if (state.nodes().getMaxNodeVersion().after(state.nodes().getSmallestNonClientNodeVersion())) { logger.debug("Skipping system indices up-to-date check as cluster has mixed versions"); return; @@ -268,14 +267,12 @@ private static boolean checkIndexMappingUpToDate(SystemIndexDescriptor descripto return false; } - // TODO[wrb]: compare _descriptor_ mapping version with mapping in cluster return Version.CURRENT.onOrBefore(readMappingVersion(descriptor, mappingMetadata)); } /** * Fetches the mapping version from an index's mapping's `_meta` info. */ - // TODO[wrb]: read the new kind of mapping version private static Version readMappingVersion(SystemIndexDescriptor descriptor, MappingMetadata mappingMetadata) { final String indexName = descriptor.getPrimaryIndex(); try { From 86d2f23956f1c6411eaf50cd1b0b69f32f34e040 Mon Sep 17 00:00:00 2001 From: William Brafford Date: Tue, 15 Aug 2023 14:52:09 -0400 Subject: [PATCH 34/34] Remove TODOs tracked elsewhere pt 2 --- .../xpack/core/security/test/TestRestrictedIndices.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java index afa9ca5af4239..92c3a65ffeaa3 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/test/TestRestrictedIndices.java @@ -265,7 +265,7 @@ private static XContentBuilder mockMappings() { XContentBuilder builder = jsonBuilder().startObject() .startObject(SINGLE_MAPPING_NAME) .startObject("_meta") - .field("version", Version.CURRENT) // TODO[wrb]: remove + .field("version", Version.CURRENT) .field(SystemIndexDescriptor.VERSION_META_KEY, 0) .endObject() .field("dynamic", "strict")