Add mappings version number to SystemIndexDescriptor

modules/ingest-geoip/src/main/java/org/elasticsearch/ingest/geoip/IngestGeoIpPlugin.java

@@ -73,6 +73,7 @@
 
 public class IngestGeoIpPlugin extends Plugin implements IngestPlugin, SystemIndexPlugin, Closeable, PersistentTaskPlugin, ActionPlugin {
     public static final Setting<Long> CACHE_SIZE = Setting.longSetting("ingest.geoip.cache_size", 1000, 0, Setting.Property.NodeScope);
+    private static final int GEOIP_INDEX_MAPPINGS_VERSION = 1;
 
     static Set<String> DEFAULT_DATABASE_FILENAMES = Set.of("GeoLite2-ASN.mmdb", "GeoLite2-City.mmdb", "GeoLite2-Country.mmdb");
 
@@ -225,6 +226,7 @@ private static XContentBuilder mappings() {
                 .startObject(SINGLE_MAPPING_NAME)
                 .startObject("_meta")
                 .field("version", Version.CURRENT)
+                .field(SystemIndexDescriptor.VERSION_META_KEY, GEOIP_INDEX_MAPPINGS_VERSION)
                 .endObject()
                 .field("dynamic", "strict")
                 .startObject("properties")

modules/reindex/src/internalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java

@@ -209,6 +209,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna
             {
                 builder.startObject("_meta");
                 builder.field(VERSION_META_KEY, META_VERSION);
+                builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1);
                 builder.field(DESCRIPTOR_MANAGED_META_KEY, descriptorManaged);
                 builder.field(DESCRIPTOR_INTERNAL_META_KEY, descriptorInternal);
                 builder.endObject();

qa/smoke-test-http/src/javaRestTest/java/org/elasticsearch/http/SystemIndexRestIT.java

@@ -169,6 +169,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
                 {
                     builder.startObject("_meta");
                     builder.field("version", Version.CURRENT.toString());
+                    builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1);
                     builder.endObject();
 
                     builder.field("dynamic", "strict");

qa/system-indices/src/main/java/org/elasticsearch/system/indices/SystemIndicesQA.java

@@ -111,6 +111,7 @@ private static XContentBuilder mappings() {
                 .startObject(SINGLE_MAPPING_NAME)
                 .startObject("_meta")
                 .field("version", Version.CURRENT)
+                .field(SystemIndexDescriptor.VERSION_META_KEY, 1)
                 .endObject()
                 .field("dynamic", "strict")
                 .startObject("properties")

server/src/internalClusterTest/java/org/elasticsearch/action/admin/indices/create/CreateSystemIndicesIT.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.cluster.metadata.Template;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.indices.TestSystemIndexDescriptor;
 import org.elasticsearch.indices.TestSystemIndexDescriptorAllowsTemplates;
 import org.elasticsearch.indices.TestSystemIndexPlugin;
@@ -38,8 +39,6 @@
 import org.junit.After;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -51,7 +50,6 @@
 
 import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
 import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
-import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -352,7 +350,6 @@ private void assertHasAliases(Set<String> aliasNames, String name, String primar
 
     /**
      * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
-     * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
      */
     private void assertMappingsAndSettings(String expectedMappings, String concreteIndex) {
         final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME))
@@ -366,11 +363,7 @@ private void assertMappingsAndSettings(String expectedMappings, String concreteI
         );
         final Map<String, Object> sourceAsMap = mappings.get(concreteIndex).getSourceAsMap();
 
-        try {
-            assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+        assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false)));
 
         final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME))
             .actionGet();

server/src/internalClusterTest/java/org/elasticsearch/aliases/NetNewSystemIndexAliasIT.java

@@ -68,6 +68,7 @@ public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings sett
                 {
                     builder.startObject("_meta");
                     builder.field("version", Version.CURRENT.toString());
+                    builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1);
                     builder.endObject();
 
                     builder.field("dynamic", "strict");

server/src/internalClusterTest/java/org/elasticsearch/indices/SystemIndexMappingUpdateServiceIT.java

@@ -17,21 +17,19 @@
 import org.elasticsearch.cluster.metadata.MappingMetadata;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.CollectionUtils;
+import org.elasticsearch.common.xcontent.XContentHelper;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.test.ESIntegTestCase;
 import org.elasticsearch.xcontent.XContentType;
 import org.junit.Before;
 
-import java.io.IOException;
-import java.io.UncheckedIOException;
 import java.util.Collection;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
 import static org.elasticsearch.indices.TestSystemIndexDescriptor.INDEX_NAME;
 import static org.elasticsearch.indices.TestSystemIndexDescriptor.PRIMARY_INDEX_NAME;
-import static org.elasticsearch.test.XContentTestUtils.convertToXContent;
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 
@@ -122,7 +120,6 @@ private void triggerClusterStateUpdates() {
 
     /**
      * Fetch the mappings and settings for {@link TestSystemIndexDescriptor#INDEX_NAME} and verify that they match the expected values.
-     * Note that in the case of the mappings, this is just a dumb string comparison, so order of keys matters.
      */
     private void assertMappingsAndSettings(String expectedMappings) {
         final GetMappingsResponse getMappingsResponse = indicesAdmin().getMappings(new GetMappingsRequest().indices(INDEX_NAME))
@@ -136,11 +133,7 @@ private void assertMappingsAndSettings(String expectedMappings) {
         );
         final Map<String, Object> sourceAsMap = mappings.get(PRIMARY_INDEX_NAME).getSourceAsMap();
 
-        try {
-            assertThat(convertToXContent(sourceAsMap, XContentType.JSON).utf8ToString(), equalTo(expectedMappings));
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
+        assertThat(sourceAsMap, equalTo(XContentHelper.convertToMap(XContentType.JSON.xContent(), expectedMappings, false)));
 
         final GetSettingsResponse getSettingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(INDEX_NAME))
             .actionGet();

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -97,6 +97,7 @@ public static String getOldMappings() {
             builder.startObject();
             {
                 builder.startObject("_meta");
+                builder.field(SystemIndexDescriptor.VERSION_META_KEY, 0);
                 builder.field("version", Version.CURRENT.previousMajor().toString());
                 builder.endObject();
 
@@ -123,6 +124,7 @@ public static String getNewMappings() {
             builder.startObject();
             {
                 builder.startObject("_meta");
+                builder.field(SystemIndexDescriptor.VERSION_META_KEY, 1);
                 builder.field("version", Version.CURRENT.toString());
                 builder.endObject();
 

server/src/main/java/org/elasticsearch/indices/SystemIndexDescriptor.java

@@ -12,6 +12,8 @@
 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
 import org.apache.lucene.util.automaton.Operations;
 import org.elasticsearch.Version;
+import org.elasticsearch.action.admin.indices.create.AutoCreateAction;
+import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
 import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.cluster.metadata.Metadata;
 import org.elasticsearch.cluster.metadata.SystemIndexMetadataUpgradeService;
@@ -130,18 +132,25 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
 
     /**
      * For internally-managed indices, specifies a key name under <code>_meta</code> in the index mappings
-     * that contains the index's mappings' version.
+     * that contains the index's mappings' {@link Version}. We need to read and write this field for
+     * backwards compatibility.
      */
-    private final String versionMetaKey;
+    private final String mappingsNodeVersionMetaKey;
+
+    /** The version meta key for the integer system index mapping version */
+    public static final String VERSION_META_KEY = "managed_index_mappings_version";
 
     /** For internally-managed indices, specifies the origin to use when creating or updating the index */
     private final String origin;
 
     /** The minimum cluster node version required for this descriptor */
     private final Version minimumNodeVersion;
 
+    /** Legacy mapping version from the descriptor */
+    private final Version mappingsNodeVersion;
+
     /** Mapping version from the descriptor */
-    private final Version mappingVersion;
+    private final MappingsVersion mappingsVersion;
 
     /** Whether there are dynamic fields in this descriptor's mappings */
     private final boolean hasDynamicMappings;
@@ -189,7 +198,7 @@ public class SystemIndexDescriptor implements IndexPatternMatcher, Comparable<Sy
      * @param settings The settings to apply to this index when auto-creating, if appropriate
      * @param aliasName An alias for the index, or null
      * @param indexFormat A value for the `index.format` setting. Pass 0 or higher.
-     * @param versionMetaKey a mapping key under <code>_meta</code> where a version can be found, which indicates the
+     * @param mappingsNodeVersionMetaKey a mapping key under <code>_meta</code> where a version can be found, which indicates the
     *                       Elasticsearch version when the index was created.
      * @param origin the client origin to use when creating this index. Internal system indices must not provide an origin, while external
      *               system indices must do so.
@@ -209,7 +218,7 @@ protected SystemIndexDescriptor(
         Settings settings,
         String aliasName,
         int indexFormat,
-        String versionMetaKey,
+        String mappingsNodeVersionMetaKey,
         String origin,
         Version minimumNodeVersion,
         Type type,
@@ -262,18 +271,22 @@ protected SystemIndexDescriptor(
             Objects.requireNonNull(settings, "Must supply settings for a managed system index");
             Strings.requireNonEmpty(mappings, "Must supply mappings for a managed system index");
             Strings.requireNonEmpty(primaryIndex, "Must supply primaryIndex for a managed system index");
-            Strings.requireNonEmpty(versionMetaKey, "Must supply versionMetaKey for a managed system index");
+            Strings.requireNonEmpty(mappingsNodeVersionMetaKey, "Must supply nodeVersionMetaKey for a managed system index");
             Strings.requireNonEmpty(origin, "Must supply origin for a managed system index");
             if (settings.getAsInt(IndexMetadata.INDEX_FORMAT_SETTING.getKey(), 0) != indexFormat) {
                 throw new IllegalArgumentException("Descriptor index format does not match index format in managed settings");
             }
-            this.mappingVersion = extractVersionFromMappings(mappings, versionMetaKey);
+            this.mappingsNodeVersion = extractNodeVersionFromMappings(mappings, mappingsNodeVersionMetaKey);
+            this.mappingsVersion = extractVersionFromMappings(mappings);
+            assert mappingsVersion.version >= 0 : "The mappings version must not be negative";
+
         } else {
             assert Objects.isNull(settings) : "Unmanaged index descriptors should not have settings";
             assert Objects.isNull(mappings) : "Unmanaged index descriptors should not have mappings";
             assert Objects.isNull(primaryIndex) : "Unmanaged index descriptors should not have a primary index";
-            assert Objects.isNull(versionMetaKey) : "Unmanaged index descriptors should not have a version meta key";
-            this.mappingVersion = null;
+            assert Objects.isNull(mappingsNodeVersionMetaKey) : "Unmanaged index descriptors should not have a version meta key";
+            this.mappingsNodeVersion = null;
+            this.mappingsVersion = null;
         }
 
         Objects.requireNonNull(allowedElasticProductOrigins, "allowedProductOrigins must not be null");
@@ -355,7 +368,7 @@ protected SystemIndexDescriptor(
             throw new IllegalArgumentException("System indices must have " + IndexMetadata.SETTING_INDEX_HIDDEN + " set to true.");
         }
         this.indexFormat = indexFormat;
-        this.versionMetaKey = versionMetaKey;
+        this.mappingsNodeVersionMetaKey = mappingsNodeVersionMetaKey;
         this.origin = origin;
         this.minimumNodeVersion = minimumNodeVersion;
         this.type = type;
@@ -450,9 +463,9 @@ public int getIndexFormat() {
         return this.indexFormat;
     }
 
-    public String getVersionMetaKey() {
+    public String getMappingsNodeVersionMetaKey() {
         assert isAutomaticallyManaged() : "Do not request version meta keys for unmanaged system indices";
-        return this.versionMetaKey;
+        return this.mappingsNodeVersionMetaKey;
     }
 
     public Version getMinimumNodeVersion() {
@@ -506,11 +519,23 @@ public boolean allowsTemplates() {
         return allowsTemplates;
     }
 
-    public Version getMappingVersion() {
+    /**
+     * Use of the mappings {@link Version} should be replaced with the value returned from {@link #getMappingsVersion()}
+     * @return Elasticsearch version associated with this descriptor's mappings.
+     */
+    @Deprecated
+    public Version getMappingsNodeVersion() {
+        if (isAutomaticallyManaged() == false) {
+            throw new IllegalStateException(this + " is not managed so there are no mappings or version");
+        }
+        return mappingsNodeVersion;
+    }
+
+    public MappingsVersion getMappingsVersion() {
         if (isAutomaticallyManaged() == false) {
             throw new IllegalStateException(this + " is not managed so there are no mappings or version");
         }
-        return mappingVersion;
+        return mappingsVersion;
     }
 
     /**
@@ -612,6 +637,20 @@ public boolean isInternal() {
         }
     }
 
+    /**
+     * The version of the mapping, which should be stored as an int in a mapping metadata
+     * field. This will be used with prior index descriptors to determine which mappings
+     * should be used when creating a system index, and it will be used to determine when
+     * mappings should be updated with the latest mappings from the system index descriptor.
+     * See {@link SystemIndexMappingUpdateService}, {@link TransportCreateIndexAction},
+     * and {@link AutoCreateAction}.
+     * <p>
+     * Version should be a non-negative integer for a managed index, or -1 for unmanaged indices.
+     * The hash is a hash of the system index descriptor's mappings so that we can warn
+     * in case of inconsistencies across nodes.
+     */
+    public record MappingsVersion(int version, int hash) {};
+
     /**
      * Provides a fluent API for building a {@link SystemIndexDescriptor}. Validation still happens in that class.
      */
@@ -818,8 +857,31 @@ static boolean findDynamicMapping(Map<String, Object> map) {
         return false;
     }
 
+    @SuppressWarnings("unchecked") // we do a lot of casting of maps
+    private static MappingsVersion extractVersionFromMappings(String mappings) {
+        final Map<String, Object> mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, true);
+        final Map<String, Object> doc = (Map<String, Object>) mappingsMap.get("_doc");
+        final Map<String, Object> meta;
+        final Map<String, Object> properties;
+        if (doc == null) {
+            meta = (Map<String, Object>) mappingsMap.get("_meta");
+            properties = (Map<String, Object>) mappingsMap.get("properties");
+        } else {
+            meta = (Map<String, Object>) doc.get("_meta");
+            properties = (Map<String, Object>) doc.get("properties");
+        }
+        if (meta == null) {
+            throw new IllegalStateException("mappings do not have _meta field");
+        }
+        final Integer value = (Integer) meta.get(VERSION_META_KEY);
+        if (value == null) {
+            throw new IllegalArgumentException("mappings do not have a version in _meta." + VERSION_META_KEY);
+        }
+        return new MappingsVersion(value, Objects.hash(properties));
+    }
+
     @SuppressWarnings("unchecked")
-    private static Version extractVersionFromMappings(String mappings, String versionMetaKey) {
+    private static Version extractNodeVersionFromMappings(String mappings, String versionMetaKey) {
         final Map<String, Object> mappingsMap = XContentHelper.convertToMap(XContentType.JSON.xContent(), mappings, false);
         final Map<String, Object> doc = (Map<String, Object>) mappingsMap.get("_doc");
         final Map<String, Object> meta;

server/src/main/java/org/elasticsearch/indices/SystemIndexMappingUpdateService.java

@@ -289,15 +289,18 @@ private static Version readMappingVersion(SystemIndexDescriptor descriptor, Mapp
                 return Version.V_EMPTY;
             }
 
-            final Object rawVersion = meta.get(descriptor.getVersionMetaKey());
+            final Object rawVersion = meta.get(descriptor.getMappingsNodeVersionMetaKey());
             if (rawVersion instanceof Integer) {
                 // This can happen with old system indices, such as .tasks, which were created before we used an Elasticsearch
                 // version here. We should just replace the template to be sure.
                 return Version.V_EMPTY;
             }
             final String versionString = rawVersion != null ? rawVersion.toString() : null;
             if (versionString == null) {
-                logger.warn("No value found in mappings for [_meta.{}], assuming mappings update required", descriptor.getVersionMetaKey());
+                logger.warn(
+                    "No value found in mappings for [_meta.{}], assuming mappings update required",
+                    descriptor.getMappingsNodeVersionMetaKey()
+                );
                 // If we called `Version.fromString(null)`, it would return `Version.CURRENT` and we wouldn't update the mappings
                 return Version.V_EMPTY;
             }

server/src/main/java/org/elasticsearch/synonyms/SynonymsManagementAPIService.java

@@ -81,6 +81,7 @@ public class SynonymsManagementAPIService {
     public static final int MAX_SYNONYMS_SETS = 10_000;
     private static final String SYNONYM_RULE_ID_FIELD = SynonymRule.ID_FIELD.getPreferredName();
     private static final String SYNONYM_SETS_AGG_NAME = "synonym_sets_aggr";
+    private static final int SYNONYMS_INDEX_MAPPINGS_VERSION = 1;
 
     private final Client client;
 
@@ -123,6 +124,7 @@ private static XContentBuilder mappings() {
                     builder.startObject("_meta");
                     {
                         builder.field("version", Version.CURRENT.toString());
+                        builder.field(SystemIndexDescriptor.VERSION_META_KEY, SYNONYMS_INDEX_MAPPINGS_VERSION);
                     }
                     builder.endObject();
                     builder.field("dynamic", "strict");