Unify AuthC chain - 1. Authenticator extraction (#77293)

This PR is a first step to refactor and unify the authentication chain. An
Authenticator interface is extracted for following concrete Authenticators
(listed in decreasing priority):

* Service token
* OAuth2 token
* API key
* Realms

Before runnning the above authenticators, existing authentication from
ThreadContext is always checked first. After running the authenticators,
fallback, anonymous and lookup users are checked more consistently. Failed
authentication with either OAuth2 token or API key now reports the attempted
and failed credentials instead of "missing credentials".

In above authenticators, the RealmsAuthenticator has its own sub-chain.
Technically, this sub-chain can also be flattend and be part of the main chain.
But it's impractical to do so without also changing the existing behaviour.
Though the change makes sense imo, it adds a lot complexities of the PR. So it
is better to be left as future work.

Relates: #75607

Co-authored-by: Tim Vernum <tim@adjective.org>

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/AuthenticationResult.java

@@ -80,7 +80,6 @@ public Map<String, Object> getMetadata() {
      * @param user The user that was authenticated. Cannot be {@code null}.
      */
     public static AuthenticationResult success(User user) {
-        Objects.requireNonNull(user);
         return success(user, null);
     }
 
@@ -90,6 +89,7 @@ public static AuthenticationResult success(User user) {
      * @see #success(User)
      */
     public static AuthenticationResult success(User user, @Nullable Map<String, Object> metadata) {
+        Objects.requireNonNull(user);
         return new AuthenticationResult(Status.SUCCESS, user, null, null, metadata);
     }
 

x-pack/plugin/security/qa/security-basic/src/javaRestTest/java/org/elasticsearch/xpack/security/SecurityWithBasicLicenseIT.java

@@ -218,7 +218,8 @@ private void assertAuthenticateWithToken(String accessToken, boolean shouldSucce
         } else {
             ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
             assertThat(e.getResponse().getStatusLine().getStatusCode(), equalTo(401));
-            assertThat(e.getMessage(), containsString("missing authentication credentials for REST request"));
+            assertThat(e.getMessage(), containsString(
+                "unable to authenticate with provided credentials and anonymous access is not allowed for this request"));
         }
     }
 

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyAuthenticator.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.security.authc;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.xpack.core.security.authc.Authentication;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
+import org.elasticsearch.xpack.core.security.support.Exceptions;
+import org.elasticsearch.xpack.security.authc.ApiKeyService.ApiKeyCredentials;
+
+class ApiKeyAuthenticator implements Authenticator {
+
+    private static final Logger logger = LogManager.getLogger(ApiKeyAuthenticator.class);
+
+    private final ApiKeyService apiKeyService;
+    private final String nodeName;
+
+    ApiKeyAuthenticator(ApiKeyService apiKeyService, String nodeName) {
+        this.apiKeyService = apiKeyService;
+        this.nodeName = nodeName;
+    }
+
+    @Override
+    public String name() {
+        return "API key";
+    }
+
+    @Override
+    public AuthenticationToken extractCredentials(Context context) {
+        return apiKeyService.getCredentialsFromHeader(context.getThreadContext());
+    }
+
+    @Override
+    public void authenticate(Context context, ActionListener<Result> listener) {
+        final AuthenticationToken authenticationToken = context.getMostRecentAuthenticationToken();
+        if (false == authenticationToken instanceof ApiKeyCredentials) {
+            listener.onResponse(Authenticator.Result.notHandled());
+            return;
+        }
+        ApiKeyCredentials apiKeyCredentials = (ApiKeyCredentials) authenticationToken;
+        apiKeyService.tryAuthenticate(context.getThreadContext(), apiKeyCredentials, ActionListener.wrap(authResult -> {
+            if (authResult.isAuthenticated()) {
+                final Authentication authentication = apiKeyService.createApiKeyAuthentication(authResult, nodeName);
+                listener.onResponse(Authenticator.Result.success(authentication));
+            } else if (authResult.getStatus() == AuthenticationResult.Status.TERMINATE) {
+                Exception e = (authResult.getException() != null) ?
+                    authResult.getException() :
+                    Exceptions.authenticationError(authResult.getMessage());
+                logger.debug(new ParameterizedMessage("API key service terminated authentication for request [{}]", context.getRequest()),
+                    e);
+                listener.onFailure(e);
+            } else {
+                if (authResult.getMessage() != null) {
+                    if (authResult.getException() != null) {
+                        logger.warn(new ParameterizedMessage("Authentication using apikey failed - {}", authResult.getMessage()),
+                            authResult.getException());
+                    } else {
+                        logger.warn("Authentication using apikey failed - {}", authResult.getMessage());
+                    }
+                }
+                listener.onResponse(Authenticator.Result.unsuccessful(
+                    authResult.getMessage(),
+                    authResult.getException()));
+            }
+        }, e -> listener.onFailure(context.getRequest().exceptionProcessingRequest(e, null))));
+    }
+}

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -88,6 +88,7 @@
 import org.elasticsearch.xpack.core.security.authc.Authentication;
 import org.elasticsearch.xpack.core.security.authc.Authentication.RealmRef;
 import org.elasticsearch.xpack.core.security.authc.AuthenticationResult;
+import org.elasticsearch.xpack.core.security.authc.AuthenticationToken;
 import org.elasticsearch.xpack.core.security.authc.service.ServiceAccountSettings;
 import org.elasticsearch.xpack.core.security.authc.support.Hasher;
 import org.elasticsearch.xpack.core.security.authz.RoleDescriptor;
@@ -376,37 +377,21 @@ XContentBuilder newDocument(char[] apiKeyHashChars, String name, Authentication
         return builder;
     }
 
-    /**
-     * Checks for the presence of a {@code Authorization} header with a value that starts with
-     * {@code ApiKey }. If found this will attempt to authenticate the key.
-     */
-    void authenticateWithApiKeyIfPresent(ThreadContext ctx, ActionListener<AuthenticationResult> listener) {
-        if (isEnabled()) {
-            final ApiKeyCredentials credentials;
-            try {
-                credentials = getCredentialsFromHeader(ctx);
-            } catch (IllegalArgumentException iae) {
-                listener.onResponse(AuthenticationResult.unsuccessful(iae.getMessage(), iae));
-                return;
-            }
-
-            if (credentials != null) {
-                loadApiKeyAndValidateCredentials(ctx, credentials, ActionListener.wrap(
-                    response -> {
-                        credentials.close();
-                        listener.onResponse(response);
-                    },
-                    e -> {
-                        credentials.close();
-                        listener.onFailure(e);
-                    }
-                ));
-            } else {
-                listener.onResponse(AuthenticationResult.notHandled());
-            }
-        } else {
+    void tryAuthenticate(ThreadContext ctx, ApiKeyCredentials credentials, ActionListener<AuthenticationResult> listener) {
+        if (false == isEnabled()) {
             listener.onResponse(AuthenticationResult.notHandled());
         }
+        assert credentials != null : "api key credentials must not be null";
+        loadApiKeyAndValidateCredentials(ctx, credentials, ActionListener.wrap(
+            response -> {
+                credentials.close();
+                listener.onResponse(response);
+            },
+            e -> {
+                credentials.close();
+                listener.onFailure(e);
+            }
+        ));
     }
 
     public Authentication createApiKeyAuthentication(AuthenticationResult authResult, String nodeName) {
@@ -737,11 +722,13 @@ void validateApiKeyExpiration(ApiKeyDoc apiKeyDoc, ApiKeyCredentials credentials
      * Gets the API Key from the <code>Authorization</code> header if the header begins with
      * <code>ApiKey </code>
      */
-    static ApiKeyCredentials getCredentialsFromHeader(ThreadContext threadContext) {
-        String header = threadContext.getHeader("Authorization");
-        if (Strings.hasText(header) && header.regionMatches(true, 0, "ApiKey ", 0, "ApiKey ".length())
-            && header.length() > "ApiKey ".length()) {
-            final byte[] decodedApiKeyCredBytes = Base64.getDecoder().decode(header.substring("ApiKey ".length()));
+    ApiKeyCredentials getCredentialsFromHeader(ThreadContext threadContext) {
+        if (false == isEnabled()) {
+            return null;
+        }
+        final SecureString apiKeyString = Authenticator.extractCredentialFromAuthorizationHeader(threadContext, "ApiKey");
+        if (apiKeyString != null) {
+            final byte[] decodedApiKeyCredBytes = Base64.getDecoder().decode(CharArrays.toUtf8Bytes(apiKeyString.getChars()));
             char[] apiKeyCredChars = null;
             try {
                 apiKeyCredChars = CharArrays.utf8BytesToChars(decodedApiKeyCredBytes);
@@ -814,7 +801,7 @@ public void ensureEnabled() {
     }
 
     // public class for testing
-    public static final class ApiKeyCredentials implements Closeable {
+    public static final class ApiKeyCredentials implements AuthenticationToken, Closeable {
         private final String id;
         private final SecureString key;
 
@@ -835,6 +822,21 @@ SecureString getKey() {
         public void close() {
             key.close();
         }
+
+        @Override
+        public String principal() {
+            return id;
+        }
+
+        @Override
+        public Object credentials() {
+            return key;
+        }
+
+        @Override
+        public void clearCredentials() {
+            close();
+        }
     }
 
     private static class ApiKeyLoggingDeprecationHandler implements DeprecationHandler {