Support API keys as authentication subjects for RCS 2.0 (#93414)

This PR enables support for defining remote_indices for API keys in order to
be able to use them to authenticate cross cluster calls.

The main change is made to the role classes (SimpleRole and LimitedRole),
which now support constructing RoleDescriptorsIntersection on the querying
cluster side. This RoleDescriptorsIntersection is sent to the fulfilling cluster as
part of the RemoteAccessAuthentication.

Relates to #90614

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/Authentication.java

@@ -32,6 +32,7 @@
 import org.elasticsearch.xpack.core.security.authc.file.FileRealmSettings;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
 import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
 import org.elasticsearch.xpack.core.security.user.AnonymousUser;
 import org.elasticsearch.xpack.core.security.user.AsyncSearchUser;
 import org.elasticsearch.xpack.core.security.user.SecurityProfileUser;
@@ -49,6 +50,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import static org.elasticsearch.common.Strings.EMPTY_ARRAY;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
@@ -106,6 +108,7 @@ public final class Authentication implements ToXContentObject {
     public static final TransportVersion VERSION_API_KEY_ROLES_AS_BYTES = TransportVersion.V_7_9_0;
     public static final TransportVersion VERSION_REALM_DOMAINS = TransportVersion.V_8_2_0;
     public static final TransportVersion VERSION_METADATA_BEYOND_GENERIC_MAP = TransportVersion.V_8_8_0;
+    public static final TransportVersion VERSION_API_KEYS_WITH_REMOTE_INDICES = TransportVersion.V_8_8_0;
     private final AuthenticationType type;
     private final Subject authenticatingSubject;
     private final Subject effectiveSubject;
@@ -1066,9 +1069,25 @@ private static Map<String, Object> maybeRewriteMetadataForApiKeyRoleDescriptors(
                 : "metadata must contain role descriptor for API key authentication";
             assert metadata.containsKey(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
                 : "metadata must contain limited role descriptor for API key authentication";
+            if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_API_KEYS_WITH_REMOTE_INDICES)
+                && streamVersion.before(VERSION_API_KEYS_WITH_REMOTE_INDICES)) {
+                metadata = new HashMap<>(metadata);
+                metadata.put(
+                    AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
+                    maybeRemoveRemoteIndicesFromRoleDescriptors(
+                        (BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY)
+                    )
+                );
+                metadata.put(
+                    AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY,
+                    maybeRemoveRemoteIndicesFromRoleDescriptors(
+                        (BytesReference) metadata.get(AuthenticationField.API_KEY_LIMITED_ROLE_DESCRIPTORS_KEY)
+                    )
+                );
+            }
             if (authentication.getEffectiveSubject().getTransportVersion().onOrAfter(VERSION_API_KEY_ROLES_AS_BYTES)
                 && streamVersion.before(VERSION_API_KEY_ROLES_AS_BYTES)) {
-                metadata = new HashMap<>(metadata);
+                metadata = metadata instanceof HashMap ? metadata : new HashMap<>(metadata);
                 metadata.put(
                     AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY,
                     convertRoleDescriptorsBytesToMap((BytesReference) metadata.get(AuthenticationField.API_KEY_ROLE_DESCRIPTORS_KEY))
@@ -1131,6 +1150,32 @@ private static BytesReference convertRoleDescriptorsMapToBytes(Map<String, Objec
         }
     }
 
+    static BytesReference maybeRemoveRemoteIndicesFromRoleDescriptors(BytesReference roleDescriptorsBytes) {
+        if (roleDescriptorsBytes == null || roleDescriptorsBytes.length() == 0) {
+            return roleDescriptorsBytes;
+        }
+
+        final Map<String, Object> roleDescriptorsMap = convertRoleDescriptorsBytesToMap(roleDescriptorsBytes);
+        final AtomicBoolean removedAtLeastOne = new AtomicBoolean(false);
+        roleDescriptorsMap.entrySet().stream().forEach(entry -> {
+            if (entry.getValue() instanceof Map) {
+                @SuppressWarnings("unchecked")
+                Map<String, Object> roleDescriptor = (Map<String, Object>) entry.getValue();
+                boolean removed = roleDescriptor.remove(RoleDescriptor.Fields.REMOTE_INDICES.getPreferredName()) != null;
+                if (removed) {
+                    removedAtLeastOne.set(true);
+                }
+            }
+        });
+
+        if (removedAtLeastOne.get()) {
+            return convertRoleDescriptorsMapToBytes(roleDescriptorsMap);
+        } else {
+            // No need to serialize if we did not remove anything.
+            return roleDescriptorsBytes;
+        }
+    }
+
     static boolean equivalentRealms(String name1, String type1, String name2, String type2) {
         if (false == type1.equals(type2)) {
             return false;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/RoleDescriptor.java

@@ -357,14 +357,6 @@ public void writeTo(StreamOutput out) throws IOException {
         ConfigurableClusterPrivileges.writeArray(out, getConditionalClusterPrivileges());
         if (out.getTransportVersion().onOrAfter(TRANSPORT_VERSION_REMOTE_INDICES)) {
             out.writeArray(remoteIndicesPrivileges);
-        } else if (hasRemoteIndicesPrivileges()) {
-            throw new IllegalArgumentException(
-                "versions of Elasticsearch before ["
-                    + TRANSPORT_VERSION_REMOTE_INDICES
-                    + "] can't handle remote indices privileges and attempted to send to ["
-                    + out.getTransportVersion()
-                    + "]"
-            );
         }
     }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java

@@ -12,13 +12,18 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.transport.TransportRequest;
 import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 import org.elasticsearch.xpack.core.security.support.Automatons;
 
+import java.util.ArrayList;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
@@ -118,6 +123,27 @@ public IndicesAccessControl authorize(
         return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl);
     }
 
+    @Override
+    public RoleDescriptorsIntersection getRemoteAccessRoleDescriptorsIntersection(final String remoteClusterAlias) {
+        final RoleDescriptorsIntersection baseIntersection = baseRole.getRemoteAccessRoleDescriptorsIntersection(remoteClusterAlias);
+        // Intersecting with empty descriptors list should result in an empty intersection.
+        if (baseIntersection.roleDescriptorsList().isEmpty()) {
+            return RoleDescriptorsIntersection.EMPTY;
+        }
+        final RoleDescriptorsIntersection limitedByIntersection = limitedByRole.getRemoteAccessRoleDescriptorsIntersection(
+            remoteClusterAlias
+        );
+        if (limitedByIntersection.roleDescriptorsList().isEmpty()) {
+            return RoleDescriptorsIntersection.EMPTY;
+        }
+        final List<Set<RoleDescriptor>> mergedIntersection = new ArrayList<>(
+            baseIntersection.roleDescriptorsList().size() + limitedByIntersection.roleDescriptorsList().size()
+        );
+        mergedIntersection.addAll(baseIntersection.roleDescriptorsList());
+        mergedIntersection.addAll(limitedByIntersection.roleDescriptorsList());
+        return new RoleDescriptorsIntersection(Collections.unmodifiableList(mergedIntersection));
+    }
+
     /**
      * @return A predicate that will match all the indices that this role and the limited by role has the privilege for executing the given
      * action on.

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.RestrictedIndices;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilege;
@@ -40,6 +41,9 @@
 
 public interface Role {
 
+    // TODO move once we have a dedicated class for RCS 2.0 constants
+    String REMOTE_USER_ROLE_NAME = "_remote_user";
+
     Role EMPTY = builder(new RestrictedIndices(Automatons.EMPTY)).build();
 
     String[] names();
@@ -160,6 +164,15 @@ IndicesAccessControl authorize(
         FieldPermissionsCache fieldPermissionsCache
     );
 
+    /**
+     * Returns the intersection of role descriptors defined for a remote cluster with the given alias.
+     *
+     * @param remoteClusterAlias the remote cluster alias for which to return a role descriptors intersection
+     * @return an intersection of defined role descriptors for the remote access to a given cluster,
+     *         otherwise an empty intersection if remote privileges are not defined
+     */
+    RoleDescriptorsIntersection getRemoteAccessRoleDescriptorsIntersection(String remoteClusterAlias);
+
     /***
      * Creates a {@link LimitedRole} that uses this Role as base and the given role as limited-by.
      */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java

@@ -8,6 +8,7 @@
 
 import org.apache.lucene.util.automaton.Automaton;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
+import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.cache.Cache;
 import org.elasticsearch.common.cache.CacheBuilder;
 import org.elasticsearch.common.settings.Setting;
@@ -17,18 +18,24 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesCheckResult;
 import org.elasticsearch.xpack.core.security.authz.AuthorizationEngine.PrivilegesToCheck;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
+import org.elasticsearch.xpack.core.security.authz.RoleDescriptorsIntersection;
 import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl;
 import org.elasticsearch.xpack.core.security.authz.permission.IndicesPermission.IsResourceAuthorizedPredicate;
 import org.elasticsearch.xpack.core.security.authz.privilege.ApplicationPrivilegeDescriptor;
 import org.elasticsearch.xpack.core.security.authz.privilege.ClusterPrivilege;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
 
 public class SimpleRole implements Role {
 
@@ -168,6 +175,74 @@ public IndicesAccessControl authorize(
         return indices.authorize(action, requestedIndicesOrAliases, aliasAndIndexLookup, fieldPermissionsCache);
     }
 
+    @Override
+    public RoleDescriptorsIntersection getRemoteAccessRoleDescriptorsIntersection(final String remoteClusterAlias) {
+        final RemoteIndicesPermission remoteIndicesPermission = remoteIndices.forCluster(remoteClusterAlias);
+        if (remoteIndicesPermission.remoteIndicesGroups().isEmpty()) {
+            return RoleDescriptorsIntersection.EMPTY;
+        }
+        final List<RoleDescriptor.IndicesPrivileges> indicesPrivileges = new ArrayList<>();
+        for (RemoteIndicesPermission.RemoteIndicesGroup remoteIndicesGroup : remoteIndicesPermission.remoteIndicesGroups()) {
+            for (IndicesPermission.Group indicesGroup : remoteIndicesGroup.indicesPermissionGroups()) {
+                indicesPrivileges.add(toIndicesPrivileges(indicesGroup));
+            }
+        }
+
+        return new RoleDescriptorsIntersection(
+            new RoleDescriptor(
+                REMOTE_USER_ROLE_NAME,
+                null,
+                // The role descriptors constructed here may be cached in raw byte form, using a hash of their content as a
+                // cache key; we therefore need deterministic order when constructing them here, to ensure cache hits for
+                // equivalent role descriptors
+                indicesPrivileges.stream().sorted().toArray(RoleDescriptor.IndicesPrivileges[]::new),
+                null,
+                null,
+                null,
+                null,
+                null
+            )
+        );
+    }
+
+    private static Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> getFieldGrantExcludeGroups(IndicesPermission.Group group) {
+        if (group.getFieldPermissions().hasFieldLevelSecurity()) {
+            final List<FieldPermissionsDefinition> fieldPermissionsDefinitions = group.getFieldPermissions()
+                .getFieldPermissionsDefinitions();
+            assert fieldPermissionsDefinitions.size() == 1
+                : "a simple role can only have up to one field permissions definition per remote indices privilege";
+            final FieldPermissionsDefinition definition = fieldPermissionsDefinitions.get(0);
+            return definition.getFieldGrantExcludeGroups();
+        } else {
+            return Collections.emptySet();
+        }
+    }
+
+    private static RoleDescriptor.IndicesPrivileges toIndicesPrivileges(final IndicesPermission.Group indicesGroup) {
+        final Set<BytesReference> queries = indicesGroup.getQuery();
+        final Set<FieldPermissionsDefinition.FieldGrantExcludeGroup> fieldGrantExcludeGroups = getFieldGrantExcludeGroups(indicesGroup);
+        assert queries == null || queries.size() <= 1
+            : "translation from an indices permission group to indices privileges supports up to one DLS query but multiple queries found";
+        assert fieldGrantExcludeGroups.size() <= 1
+            : "translation from an indices permission group to indices privileges supports up to one FLS field-grant-exclude group"
+                + " but multiple groups found";
+
+        final BytesReference query = (queries == null || false == queries.iterator().hasNext()) ? null : queries.iterator().next();
+        final RoleDescriptor.IndicesPrivileges.Builder builder = RoleDescriptor.IndicesPrivileges.builder()
+            // Sort because these index privileges will be part of role descriptors that may be cached in raw byte form;
+            // we need deterministic order to ensure cache hits for equivalent role descriptors
+            .indices(Arrays.stream(indicesGroup.indices()).sorted().collect(Collectors.toList()))
+            .privileges(indicesGroup.privilege().name().stream().sorted().collect(Collectors.toList()))
+            .allowRestrictedIndices(indicesGroup.allowRestrictedIndices())
+            .query(query);
+        if (false == fieldGrantExcludeGroups.isEmpty()) {
+            final FieldPermissionsDefinition.FieldGrantExcludeGroup fieldGrantExcludeGroup = fieldGrantExcludeGroups.iterator().next();
+            builder.grantedFields(fieldGrantExcludeGroup.getGrantedFields()).deniedFields(fieldGrantExcludeGroup.getExcludedFields());
+        }
+
+        return builder.build();
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {