Cluster-state based Security role mapper

docs/changelog/107410.yaml

@@ -0,0 +1,5 @@
+pr: 107410
+summary: Cluster-state based Security role mapper
+area: Authorization
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -183,6 +183,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion INDEX_SEGMENTS_VECTOR_FORMATS = def(8_642_00_0);
     public static final TransportVersion ADD_RESOURCE_ALREADY_UPLOADED_EXCEPTION = def(8_643_00_0);
     public static final TransportVersion ESQL_MV_ORDERING_SORTED_ASCENDING = def(8_644_00_0);
+    public static final TransportVersion SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE = def(8_645_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -81,6 +81,7 @@
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExceptExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivilege;
 import org.elasticsearch.xpack.core.security.authz.privilege.ConfigurableClusterPrivileges;
 import org.elasticsearch.xpack.core.slm.SLMFeatureSetUsage;
@@ -154,6 +155,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 ConfigurableClusterPrivileges.WriteProfileDataPrivileges::createFrom
             ),
             // security : role-mappings
+            new NamedWriteableRegistry.Entry(Metadata.Custom.class, RoleMappingMetadata.TYPE, RoleMappingMetadata::new),
+            new NamedWriteableRegistry.Entry(NamedDiff.class, RoleMappingMetadata.TYPE, RoleMappingMetadata::readDiffFrom),
             new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AllExpression.NAME, AllExpression::new),
             new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
             new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java

@@ -106,6 +106,7 @@
 import org.elasticsearch.xpack.core.rest.action.RestXPackInfoAction;
 import org.elasticsearch.xpack.core.rest.action.RestXPackUsageAction;
 import org.elasticsearch.xpack.core.security.authc.TokenMetadata;
+import org.elasticsearch.xpack.core.security.authz.RoleMappingMetadata;
 import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader;
 import org.elasticsearch.xpack.core.ssl.SSLService;
 import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction;
@@ -297,6 +298,7 @@ private static boolean alreadyContainsXPackCustomMetadata(ClusterState clusterSt
         return metadata.custom(LicensesMetadata.TYPE) != null
             || metadata.custom(MlMetadata.TYPE) != null
             || metadata.custom(WatcherMetadata.TYPE) != null
+            || RoleMappingMetadata.getFromClusterState(clusterState).isEmpty() == false
             || clusterState.custom(TokenMetadata.TYPE) != null
             || metadata.custom(TransformMetadata.TYPE) != null;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/ExpressionRoleMapping.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.core.security.authc.support.mapper;
 
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.TransportVersions;
 import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
@@ -23,13 +24,15 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 import org.elasticsearch.xcontent.XContentType;
+import org.elasticsearch.xpack.core.security.authc.support.UserRoleMapper;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionParser;
 import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.RoleMapperExpression;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -39,6 +42,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.elasticsearch.common.Strings.format;
+
 /**
  * A representation of a single role-mapping for use in NativeRoleMappingStore.
  * Logically, this represents a set of roles that should be applied to any user where a boolean
@@ -69,6 +74,32 @@ public class ExpressionRoleMapping implements ToXContentObject, Writeable {
         PARSER.declareString(ignored, new ParseField(UPGRADE_API_TYPE_FIELD));
     }
 
+    /**
+     * Given the user information (in the form of {@link UserRoleMapper.UserData}) and a collection of {@link ExpressionRoleMapping}s,
+     * this returns the set of role names that should be mapped to the user, according to the provided role mapping rules.
+     */
+    public static Set<String> resolveRoles(
+        UserRoleMapper.UserData user,
+        Collection<ExpressionRoleMapping> mappings,
+        ScriptService scriptService,
+        Logger logger
+    ) {
+        ExpressionModel model = user.asModel();
+        Set<String> roles = mappings.stream()
+            .filter(ExpressionRoleMapping::isEnabled)
+            .filter(m -> m.getExpression().match(model))
+            .flatMap(m -> {
+                Set<String> roleNames = m.getRoleNames(scriptService, model);
+                logger.trace(
+                    () -> format("Applying role-mapping [{}] to user-model [{}] produced role-names [{}]", m.getName(), model, roleNames)
+                );
+                return roleNames.stream();
+            })
+            .collect(Collectors.toSet());
+        logger.debug(() -> format("Mapping user [{}] to roles [{}]", user, roles));
+        return roles;
+    }
+
     private final String name;
     private final RoleMapperExpression expression;
     private final List<String> roles;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/expressiondsl/AllExpression.java

@@ -25,7 +25,8 @@ public final class AllExpression implements RoleMapperExpression {
 
     private final List<RoleMapperExpression> elements;
 
-    AllExpression(List<RoleMapperExpression> elements) {
+    // public to be used in tests
+    public AllExpression(List<RoleMapperExpression> elements) {
         assert elements != null;
         this.elements = elements;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/mapper/expressiondsl/AnyExpression.java

@@ -25,7 +25,8 @@ public final class AnyExpression implements RoleMapperExpression {
 
     private final List<RoleMapperExpression> elements;
 
-    AnyExpression(List<RoleMapperExpression> elements) {
+    // public to be used in tests
+    public AnyExpression(List<RoleMapperExpression> elements) {
         assert elements != null;
         this.elements = elements;
     }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleMappingMetadata.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.core.security.authz;
+
+import org.elasticsearch.TransportVersion;
+import org.elasticsearch.TransportVersions;
+import org.elasticsearch.cluster.AbstractNamedDiffable;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.NamedDiff;
+import org.elasticsearch.cluster.metadata.Metadata;
+import org.elasticsearch.common.collect.Iterators;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ChunkedToXContentHelper;
+import org.elasticsearch.xcontent.ToXContent;
+import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.Set;
+
+import static org.elasticsearch.cluster.metadata.Metadata.ALL_CONTEXTS;
+
+public final class RoleMappingMetadata extends AbstractNamedDiffable<Metadata.Custom> implements Metadata.Custom {
+
+    public static final String TYPE = "role_mappings";
+
+    private static final RoleMappingMetadata EMPTY = new RoleMappingMetadata(Set.of());
+
+    public static RoleMappingMetadata getFromClusterState(ClusterState clusterState) {
+        return clusterState.metadata().custom(RoleMappingMetadata.TYPE, RoleMappingMetadata.EMPTY);
+    }
+
+    private final Set<ExpressionRoleMapping> roleMappings;
+
+    public RoleMappingMetadata(Set<ExpressionRoleMapping> roleMappings) {
+        this.roleMappings = roleMappings;
+    }
+
+    public RoleMappingMetadata(StreamInput input) throws IOException {
+        this.roleMappings = input.readCollectionAsSet(ExpressionRoleMapping::new);
+    }
+
+    public Set<ExpressionRoleMapping> getRoleMappings() {
+        return this.roleMappings;
+    }
+
+    public boolean isEmpty() {
+        return roleMappings.isEmpty();
+    }
+
+    public ClusterState updateClusterState(ClusterState clusterState) {
+        if (isEmpty()) {
+            // prefer no role mapping custom metadata to the empty role mapping metadata
+            return clusterState.copyAndUpdateMetadata(b -> b.removeCustom(RoleMappingMetadata.TYPE));
+        } else {
+            return clusterState.copyAndUpdateMetadata(b -> b.putCustom(RoleMappingMetadata.TYPE, this));
+        }
+    }
+
+    public static NamedDiff<Metadata.Custom> readDiffFrom(StreamInput streamInput) throws IOException {
+        return readDiffFrom(Metadata.Custom.class, TYPE, streamInput);
+    }
+
+    @Override
+    public Iterator<? extends ToXContent> toXContentChunked(ToXContent.Params params) {
+        return Iterators.concat(ChunkedToXContentHelper.startArray(TYPE), roleMappings.iterator(), ChunkedToXContentHelper.endArray());
+    }
+
+    @Override
+    public String getWriteableName() {
+        return TYPE;
+    }
+
+    @Override
+    public TransportVersion getMinimalSupportedVersion() {
+        return TransportVersions.SECURITY_ROLE_MAPPINGS_IN_CLUSTER_STATE;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeCollection(roleMappings);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        final var other = (RoleMappingMetadata) o;
+        return Objects.equals(roleMappings, other.roleMappings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(roleMappings);
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder builder = new StringBuilder("RoleMapping[entries=[");
+        final Iterator<ExpressionRoleMapping> entryList = roleMappings.iterator();
+        boolean firstEntry = true;
+        while (entryList.hasNext()) {
+            if (firstEntry == false) {
+                builder.append(",");
+            }
+            builder.append(entryList.next().toString());
+            firstEntry = false;
+        }
+        return builder.append("]]").toString();
+    }
+
+    @Override
+    public EnumSet<Metadata.XContentContext> context() {
+        // It is safest to have this persisted to gateway and snapshots, although maybe redundant.
+        // The persistence can become an issue in cases where {@link ReservedStateMetadata}
+        // (which records the names of the role mappings last applied) is persisted,
+        // but the role mappings themselves (stored here by the {@link RoleMappingMetadata})
+        // are not persisted.
+        return ALL_CONTEXTS;
+    }
+}

x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/authc/jwt/JwtRealmSingleNodeTests.java

@@ -169,6 +169,7 @@ protected String configRoles() {
             """;
     }
 
+    @Override
     protected boolean addMockHttpTransport() {
         return false;
     }
@@ -486,7 +487,9 @@ public void testClientSecretRotation() throws Exception {
             .expirationTime(Date.from(Instant.now().plusSeconds(600)));
         assertEquals(
             200,
-            client.performRequest(getRequest(getSignedJWT(jwt0Claims.build()), jwt0SharedSecret)).getStatusLine().getStatusCode()
+            client.performRequest(getAuthenticateRequest(getSignedJWT(jwt0Claims.build()), jwt0SharedSecret))
+                .getStatusLine()
+                .getStatusCode()
         );
         // valid jwt for realm1
         JWTClaimsSet.Builder jwt1Claims = new JWTClaimsSet.Builder();
@@ -499,7 +502,9 @@ public void testClientSecretRotation() throws Exception {
             .expirationTime(Date.from(Instant.now().plusSeconds(300)));
         assertEquals(
             200,
-            client.performRequest(getRequest(getSignedJWT(jwt1Claims.build()), jwt1SharedSecret)).getStatusLine().getStatusCode()
+            client.performRequest(getAuthenticateRequest(getSignedJWT(jwt1Claims.build()), jwt1SharedSecret))
+                .getStatusLine()
+                .getStatusCode()
         );
         // valid jwt for realm2
         JWTClaimsSet.Builder jwt2Claims = new JWTClaimsSet.Builder();
@@ -512,7 +517,9 @@ public void testClientSecretRotation() throws Exception {
             .expirationTime(Date.from(Instant.now().plusSeconds(300)));
         assertEquals(
             200,
-            client.performRequest(getRequest(getSignedJWT(jwt2Claims.build()), jwt2SharedSecret)).getStatusLine().getStatusCode()
+            client.performRequest(getAuthenticateRequest(getSignedJWT(jwt2Claims.build()), jwt2SharedSecret))
+                .getStatusLine()
+                .getStatusCode()
         );
         final PluginsService plugins = getInstanceFromNode(PluginsService.class);
         final LocalStateSecurity localStateSecurity = plugins.filterPlugins(LocalStateSecurity.class).findFirst().get();
@@ -541,30 +548,42 @@ public void testClientSecretRotation() throws Exception {
             // ensure the old value still works for realm 0 (default grace period)
             assertEquals(
                 200,
-                client.performRequest(getRequest(getSignedJWT(jwt0Claims.build()), jwt0SharedSecret)).getStatusLine().getStatusCode()
+                client.performRequest(getAuthenticateRequest(getSignedJWT(jwt0Claims.build()), jwt0SharedSecret))
+                    .getStatusLine()
+                    .getStatusCode()
             );
             assertEquals(
                 200,
-                client.performRequest(getRequest(getSignedJWT(jwt0Claims.build()), "realm0updatedSecret")).getStatusLine().getStatusCode()
+                client.performRequest(getAuthenticateRequest(getSignedJWT(jwt0Claims.build()), "realm0updatedSecret"))
+                    .getStatusLine()
+                    .getStatusCode()
             );
             // ensure the old value still works for realm 1 (explicit grace period)
             assertEquals(
                 200,
-                client.performRequest(getRequest(getSignedJWT(jwt1Claims.build()), jwt1SharedSecret)).getStatusLine().getStatusCode()
+                client.performRequest(getAuthenticateRequest(getSignedJWT(jwt1Claims.build()), jwt1SharedSecret))
+                    .getStatusLine()
+                    .getStatusCode()
             );
             assertEquals(
                 200,
-                client.performRequest(getRequest(getSignedJWT(jwt1Claims.build()), "realm1updatedSecret")).getStatusLine().getStatusCode()
+                client.performRequest(getAuthenticateRequest(getSignedJWT(jwt1Claims.build()), "realm1updatedSecret"))
+                    .getStatusLine()
+                    .getStatusCode()
             );
             // ensure the old value does not work for realm 2 (no grace period)
             ResponseException exception = expectThrows(
                 ResponseException.class,
-                () -> client.performRequest(getRequest(getSignedJWT(jwt2Claims.build()), jwt2SharedSecret)).getStatusLine().getStatusCode()
+                () -> client.performRequest(getAuthenticateRequest(getSignedJWT(jwt2Claims.build()), jwt2SharedSecret))
+                    .getStatusLine()
+                    .getStatusCode()
             );
             assertEquals(401, exception.getResponse().getStatusLine().getStatusCode());
             assertEquals(
                 200,
-                client.performRequest(getRequest(getSignedJWT(jwt2Claims.build()), "realm2updatedSecret")).getStatusLine().getStatusCode()
+                client.performRequest(getAuthenticateRequest(getSignedJWT(jwt2Claims.build()), "realm2updatedSecret"))
+                    .getStatusLine()
+                    .getStatusCode()
             );
         } finally {
             // update them back to their original values
@@ -688,7 +707,7 @@ public void testValidationDuringReloadingClientSecrets() {
         }
     }
 
-    private SignedJWT getSignedJWT(JWTClaimsSet claimsSet, byte[] hmacKeyBytes) throws Exception {
+    static SignedJWT getSignedJWT(JWTClaimsSet claimsSet, byte[] hmacKeyBytes) throws Exception {
         JWSHeader jwtHeader = new JWSHeader.Builder(JWSAlgorithm.HS256).build();
         OctetSequenceKey.Builder jwt0signer = new OctetSequenceKey.Builder(hmacKeyBytes);
         jwt0signer.algorithm(JWSAlgorithm.HS256);
@@ -701,7 +720,7 @@ private SignedJWT getSignedJWT(JWTClaimsSet claimsSet) throws Exception {
         return getSignedJWT(claimsSet, jwtHmacKey.getBytes(StandardCharsets.UTF_8));
     }
 
-    private Request getRequest(SignedJWT jwt, String sharedSecret) {
+    static Request getAuthenticateRequest(SignedJWT jwt, String sharedSecret) {
         Request request = new Request("GET", "/_security/_authenticate");
         RequestOptions.Builder options = RequestOptions.DEFAULT.toBuilder();
         options.addHeader("Authorization", "Bearer  " + jwt.serialize());
@@ -768,7 +787,7 @@ private ThreadContext prepareThreadContext(SignedJWT signedJWT, String clientSec
         return threadContext;
     }
 
-    private static GrantApiKeyRequest getGrantApiKeyForJWT(SignedJWT signedJWT, String sharedSecret) {
+    static GrantApiKeyRequest getGrantApiKeyForJWT(SignedJWT signedJWT, String sharedSecret) {
         GrantApiKeyRequest grantApiKeyRequest = new GrantApiKeyRequest();
         grantApiKeyRequest.getGrant().setType("access_token");
         grantApiKeyRequest.getGrant().setAccessToken(new SecureString(signedJWT.serialize().toCharArray()));