Move Security to use auto-managed system indices (#68375)

Backport of #67114. Part of #61656.

Change the Security plugin so that its system indices are managed automatically
by the system indices infrastructure.

Also add an `origin` field to `CreateIndexRequest` and `UpdateSettingsRequest`.

server/src/internalClusterTest/java/org/elasticsearch/indices/TestSystemIndexDescriptor.java

@@ -50,7 +50,8 @@ public class TestSystemIndexDescriptor extends SystemIndexDescriptor {
             0,
             "version",
             "stack",
-            MapperService.SINGLE_MAPPING_NAME
+            MapperService.SINGLE_MAPPING_NAME,
+            null
         );
     }
 

server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java

@@ -140,9 +140,24 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         }
 
                         final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
-                        CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
-                            ? buildSystemIndexUpdateRequest(descriptor)
-                            : buildUpdateRequest(indexName);
+                        final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
+
+                        final CreateIndexClusterStateUpdateRequest updateRequest;
+
+                        if (isSystemIndex) {
+                            final String message = descriptor.checkMinimumNodeVersion(
+                                "auto-create index",
+                                state.nodes().getMinNodeVersion()
+                            );
+                            if (message != null) {
+                                logger.warn(message);
+                                throw new IllegalStateException(message);
+                            }
+
+                            updateRequest = buildSystemIndexUpdateRequest(descriptor);
+                        } else {
+                            updateRequest = buildUpdateRequest(indexName);
+                        }
 
                         return createIndexService.applyCreateIndexRequest(currentState, updateRequest, false);
                     }

server/src/main/java/org/elasticsearch/action/admin/indices/create/CreateIndexRequest.java

@@ -76,6 +76,12 @@ public class CreateIndexRequest extends AcknowledgedRequest<CreateIndexRequest>
 
     private ActiveShardCount waitForActiveShards = ActiveShardCount.DEFAULT;
 
+    private String origin = "";
+
+    /**
+     * Constructs a new request by deserializing an input
+     * @param in the input from which to deserialize
+     */
     public CreateIndexRequest(StreamInput in) throws IOException {
         super(in);
         cause = in.readString();
@@ -107,20 +113,28 @@ public CreateIndexRequest(StreamInput in) throws IOException {
             in.readBoolean(); // updateAllTypes
         }
         waitForActiveShards = ActiveShardCount.readFrom(in);
+        if (in.getVersion().onOrAfter(Version.V_7_12_0)) {
+            origin = in.readString();
+        }
     }
 
     public CreateIndexRequest() {
     }
 
     /**
-     * Constructs a new request to create an index with the specified name.
+     * Constructs a request to create an index.
+     *
+     * @param index the name of the index
      */
     public CreateIndexRequest(String index) {
         this(index, EMPTY_SETTINGS);
     }
 
     /**
-     * Constructs a new request to create an index with the specified name and settings.
+     * Constructs a request to create an index.
+     *
+     * @param index the name of the index
+     * @param settings the settings to apply to the index
      */
     public CreateIndexRequest(String index, Settings settings) {
         this.index = index;
@@ -172,6 +186,15 @@ public String cause() {
         return cause;
     }
 
+    public String origin() {
+        return origin;
+    }
+
+    public CreateIndexRequest origin(String origin) {
+        this.origin = Objects.requireNonNull(origin);
+        return this;
+    }
+
     /**
      * The settings to create the index with.
      */
@@ -466,6 +489,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeBoolean(true); // updateAllTypes
         }
         waitForActiveShards.writeTo(out);
+        if (out.getVersion().onOrAfter(Version.V_7_12_0)) {
+            out.writeString(origin);
+        }
     }
 
     @Override

server/src/main/java/org/elasticsearch/action/admin/indices/create/TransportCreateIndexAction.java

@@ -8,6 +8,8 @@
 
 package org.elasticsearch.action.admin.indices.create;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.admin.indices.alias.Alias;
 import org.elasticsearch.action.support.ActionFilters;
@@ -19,6 +21,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataCreateIndexService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.indices.SystemIndexDescriptor;
@@ -35,6 +38,7 @@
  * Create index action.
  */
 public class TransportCreateIndexAction extends TransportMasterNodeAction<CreateIndexRequest, CreateIndexResponse> {
+    private static final Logger logger = LogManager.getLogger(TransportCreateIndexAction.class);
 
     private final MetadataCreateIndexService createIndexService;
     private final SystemIndices systemIndices;
@@ -66,9 +70,25 @@ protected void masterOperation(final CreateIndexRequest request, final ClusterSt
         final String indexName = indexNameExpressionResolver.resolveDateMathExpression(request.index());
 
         final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(indexName);
-        final CreateIndexClusterStateUpdateRequest updateRequest = descriptor != null && descriptor.isAutomaticallyManaged()
-            ? buildSystemIndexUpdateRequest(request, cause, descriptor)
-            : buildUpdateRequest(request, cause, indexName);
+        final boolean isSystemIndex = descriptor != null && descriptor.isAutomaticallyManaged();
+
+        final CreateIndexClusterStateUpdateRequest updateRequest;
+
+        // Requests that a cluster generates itself are permitted to create a system index with
+        // different mappings, settings etc. This is so that rolling upgrade scenarios still work.
+        // We check this via the request's origin. Eventually, `SystemIndexManager` will reconfigure
+        // the index to the latest settings.
+        if (isSystemIndex && Strings.isNullOrEmpty(request.origin())) {
+            final String message = descriptor.checkMinimumNodeVersion("create index", state.nodes().getMinNodeVersion());
+            if (message != null) {
+                logger.warn(message);
+                listener.onFailure(new IllegalStateException(message));
+                return;
+            }
+            updateRequest = buildSystemIndexUpdateRequest(request, cause, descriptor);
+        } else {
+            updateRequest = buildUpdateRequest(request, cause, indexName);
+        }
 
         createIndexService.createIndex(updateRequest, listener.map(response ->
             new CreateIndexResponse(response.isAcknowledged(), response.isShardsAcknowledged(), indexName)));

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java

@@ -7,6 +7,8 @@
  */
 package org.elasticsearch.action.admin.indices.mapping.put;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
 import org.elasticsearch.action.support.master.AcknowledgedResponse;
@@ -19,6 +21,7 @@
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.Index;
+import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -27,7 +30,10 @@
 
 public class TransportAutoPutMappingAction extends AcknowledgedTransportMasterNodeAction<PutMappingRequest> {
 
+    private static final Logger logger = LogManager.getLogger(TransportAutoPutMappingAction.class);
+
     private final MetadataMappingService metadataMappingService;
+    private final SystemIndices systemIndices;
 
     @Inject
     public TransportAutoPutMappingAction(
@@ -36,10 +42,12 @@ public TransportAutoPutMappingAction(
             final ThreadPool threadPool,
             final MetadataMappingService metadataMappingService,
             final ActionFilters actionFilters,
-            final IndexNameExpressionResolver indexNameExpressionResolver) {
+            final IndexNameExpressionResolver indexNameExpressionResolver,
+            final SystemIndices systemIndices) {
         super(AutoPutMappingAction.NAME, transportService, clusterService, threadPool, actionFilters,
             PutMappingRequest::new, indexNameExpressionResolver, ThreadPool.Names.SAME);
         this.metadataMappingService = metadataMappingService;
+        this.systemIndices = systemIndices;
     }
 
     @Override
@@ -61,6 +69,14 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta
     protected void masterOperation(final PutMappingRequest request, final ClusterState state,
                                    final ActionListener<AcknowledgedResponse> listener) {
         final Index[] concreteIndices = new Index[] {request.getConcreteIndex()};
+
+        final String message = TransportPutMappingAction.checkForSystemIndexViolations(systemIndices, concreteIndices, request);
+        if (message != null) {
+            logger.warn(message);
+            listener.onFailure(new IllegalStateException(message));
+            return;
+        }
+
         performMappingUpdate(concreteIndices, request, listener, metadataMappingService);
     }
 

server/src/main/java/org/elasticsearch/action/admin/indices/mapping/put/TransportPutMappingAction.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataMappingService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.index.IndexNotFoundException;
@@ -80,22 +81,17 @@ protected void masterOperation(final PutMappingRequest request, final ClusterSta
                                    final ActionListener<AcknowledgedResponse> listener) {
         try {
             final Index[] concreteIndices = resolveIndices(state, request, indexNameExpressionResolver);
-            final String mappingSource = request.source();
 
             final Optional<Exception> maybeValidationException = requestValidators.validateRequest(request, state, concreteIndices);
             if (maybeValidationException.isPresent()) {
                 listener.onFailure(maybeValidationException.get());
                 return;
             }
 
-            final List<String> violations = checkForSystemIndexViolations(concreteIndices, mappingSource);
-            if (violations.isEmpty() == false) {
-                final String message = "Cannot update mappings in "
-                    + violations
-                    + ": system indices can only use mappings from their descriptors, "
-                    + "but the mappings in the request did not match those in the descriptors(s)";
+            final String message = checkForSystemIndexViolations(systemIndices, concreteIndices, request);
+            if (message != null) {
                 logger.warn(message);
-                listener.onFailure(new IllegalArgumentException(message));
+                listener.onFailure(new IllegalStateException(message));
                 return;
             }
 
@@ -149,21 +145,36 @@ public void onFailure(Exception t) {
         });
     }
 
-    private List<String> checkForSystemIndexViolations(Index[] concreteIndices, String requestMappings) {
+    static String checkForSystemIndexViolations(SystemIndices systemIndices, Index[] concreteIndices, PutMappingRequest request) {
+        // Requests that a cluster generates itself are permitted to have a difference in mappings
+        // so that rolling upgrade scenarios still work. We check this via the request's origin.
+        if (Strings.isNullOrEmpty(request.origin()) == false) {
+            return null;
+        }
+
         List<String> violations = new ArrayList<>();
 
+        final String requestMappings = request.source();
+
         for (Index index : concreteIndices) {
             final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
             if (descriptor != null && descriptor.isAutomaticallyManaged()) {
                 final String descriptorMappings = descriptor.getMappings();
-
                 // Technically we could trip over a difference in whitespace here, but then again nobody should be trying to manually
                 // update a descriptor's mappings.
                 if (descriptorMappings.equals(requestMappings) == false) {
                     violations.add(index.getName());
                 }
             }
         }
-        return violations;
+
+        if (violations.isEmpty() == false) {
+            return "Cannot update mappings in "
+                + violations
+                + ": system indices can only use mappings from their descriptors, "
+                + "but the mappings in the request did not match those in the descriptors(s)";
+        }
+
+        return null;
     }
 }

server/src/main/java/org/elasticsearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java

@@ -8,13 +8,6 @@
 
 package org.elasticsearch.action.admin.indices.settings.put;
 
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.stream.Collectors;
-
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -29,6 +22,7 @@
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetadataUpdateSettingsService;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.Index;
@@ -37,6 +31,14 @@
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
 public class TransportUpdateSettingsAction extends AcknowledgedTransportMasterNodeAction<UpdateSettingsRequest> {
 
     private static final Logger logger = LogManager.getLogger(TransportUpdateSettingsAction.class);
@@ -78,16 +80,15 @@ protected void masterOperation(final UpdateSettingsRequest request, final Cluste
         final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
         final Settings requestSettings = request.settings();
 
-
-        final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, requestSettings);
+        final Map<String, List<String>> systemIndexViolations = checkForSystemIndexViolations(concreteIndices, request);
         if (systemIndexViolations.isEmpty() == false) {
             final String message = "Cannot override settings on system indices: "
                 + systemIndexViolations.entrySet()
                     .stream()
                     .map(entry -> "[" + entry.getKey() + "] -> " + entry.getValue())
                     .collect(Collectors.joining(", "));
             logger.warn(message);
-            listener.onFailure(new IllegalArgumentException(message));
+            listener.onFailure(new IllegalStateException(message));
             return;
         }
 
@@ -117,11 +118,18 @@ public void onFailure(Exception t) {
      * that the system index's descriptor expects.
      *
      * @param concreteIndices the indices being updated
-     * @param requestSettings the settings to be applied
+     * @param request the update request
      * @return a mapping from system index pattern to the settings whose values would be overridden. Empty if there are no violations.
      */
-    private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, Settings requestSettings) {
-        final Map<String, List<String>> violations = new HashMap<>();
+    private Map<String, List<String>> checkForSystemIndexViolations(Index[] concreteIndices, UpdateSettingsRequest request) {
+        // Requests that a cluster generates itself are permitted to have a difference in settings
+        // so that rolling upgrade scenarios still work. We check this via the request's origin.
+        if (Strings.isNullOrEmpty(request.origin()) == false) {
+            return Collections.emptyMap();
+        }
+
+        final Map<String, List<String>> violationsByIndex = new HashMap<>();
+        final Settings requestSettings = request.settings();
 
         for (Index index : concreteIndices) {
             final SystemIndexDescriptor descriptor = systemIndices.findMatchingDescriptor(index.getName());
@@ -138,10 +146,11 @@ private Map<String, List<String>> checkForSystemIndexViolations(Index[] concrete
                 }
 
                 if (failedKeys.isEmpty() == false) {
-                    violations.put(descriptor.getIndexPattern(), failedKeys);
+                    violationsByIndex.put(descriptor.getIndexPattern(), failedKeys);
                 }
             }
         }
-        return violations;
+
+        return violationsByIndex;
     }
 }