Integration tests for JWT realm (#84464)

Adds a new QA project to integration test the JWT Realm.

The new JwtRestIT is a rest based test, that covers the following
scenarios:

- Successful authentication using:
   - RSA or HMAC
   - SharedSecret client authentication, or no client authentication
   - Role mapping based authz, or delegation based authz

- Unsuccessful authentication due to:
   - Expired JWT
   - Invalid signature (RSA + HMAC)
   - Invalid client credential
   - Missing delegation user

Co-authored-by: Justin Cranford <justin.cranford@elastic.co>

x-pack/plugin/core/src/test/java/org/elasticsearch/test/TestMatchers.java

@@ -6,6 +6,8 @@
  */
 package org.elasticsearch.test;
 
+import org.elasticsearch.client.Response;
+import org.elasticsearch.rest.RestStatus;
 import org.hamcrest.BaseMatcher;
 import org.hamcrest.CoreMatchers;
 import org.hamcrest.CustomMatcher;
@@ -87,6 +89,19 @@ public static Matcher<String> matchesPattern(Pattern pattern) {
         return predicate("Matches " + pattern.pattern(), String.class, pattern.asPredicate());
     }
 
+    public static Matcher<Response> hasStatusCode(RestStatus expected) {
+        return new CustomMatcher<>("Response with status " + expected.getStatus() + " (" + expected.name() + ")") {
+            @Override
+            public boolean matches(Object item) {
+                if (item instanceof Response response) {
+                    return response.getStatusLine().getStatusCode() == expected.getStatus();
+                } else {
+                    return false;
+                }
+            }
+        };
+    }
+
     private static <T> Matcher<T> predicate(String description, Class<T> type, Predicate<T> predicate) {
         return new CustomMatcher<T>(description) {
             @Override

x-pack/plugin/security/qa/jwt-realm/build.gradle

@@ -0,0 +1,74 @@
+apply plugin: 'elasticsearch.internal-java-rest-test'
+
+dependencies {
+  javaRestTestImplementation project(path: xpackModule('core'))
+  javaRestTestImplementation(testArtifact(project(xpackModule('security'))))
+  javaRestTestImplementation(testArtifact(project(xpackModule('core'))))
+  javaRestTestImplementation project(":client:rest")
+}
+
+testClusters.matching { it.name == 'javaRestTest' }.configureEach {
+  testDistribution = 'DEFAULT'
+
+  numberOfNodes = 2
+
+  // This can be turned on to help debug failing tests, but generates more output than is needed for CI
+  // setting 'logger.org.elasticsearch.xpack.security.authc.jwt', 'DEBUG'
+
+  extraConfigFile 'http.key', file('src/javaRestTest/resources/ssl/http.key')
+  extraConfigFile 'http.crt', file('src/javaRestTest/resources/ssl/http.crt')
+  extraConfigFile 'ca.crt', file('src/javaRestTest/resources/ssl/ca.crt')
+  extraConfigFile 'rsa.jwkset', file('src/javaRestTest/resources/jwk/rsa-public-jwkset.json')
+
+  setting 'xpack.ml.enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.http.ssl.enabled', 'true'
+  setting 'xpack.security.transport.ssl.enabled', 'false'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+
+  setting 'xpack.security.http.ssl.enabled', 'true'
+  setting 'xpack.security.http.ssl.certificate', 'http.crt'
+  setting 'xpack.security.http.ssl.key', 'http.key'
+  setting 'xpack.security.http.ssl.key_passphrase', 'http-password'
+  setting 'xpack.security.http.ssl.certificate_authorities', 'ca.crt'
+
+  setting 'xpack.security.authc.realms.file.admin_file.order', '0'
+  setting 'xpack.security.authc.realms.native.lookup_native.order', '1'
+
+  // These realm settings are generated by JwtRealmGenerateTests
+  setting 'xpack.security.authc.realms.jwt.jwt1.order', '2'
+  setting 'xpack.security.authc.realms.jwt.jwt1.allowed_issuer', 'https://issuer.example.com/'
+  setting 'xpack.security.authc.realms.jwt.jwt1.allowed_audiences', 'https://audience.example.com/'
+  setting 'xpack.security.authc.realms.jwt.jwt1.claims.principal', 'sub'
+  setting 'xpack.security.authc.realms.jwt.jwt1.claims.groups', 'roles'
+  setting 'xpack.security.authc.realms.jwt.jwt1.client_authentication.type', 'None'
+  // Use default value (RS256) for signature algorithm
+  setting 'xpack.security.authc.realms.jwt.jwt1.pkc_jwkset_path', 'rsa.jwkset'
+
+  setting 'xpack.security.authc.realms.jwt.jwt2.order', '3'
+  setting 'xpack.security.authc.realms.jwt.jwt2.allowed_issuer', 'my-issuer'
+  setting 'xpack.security.authc.realms.jwt.jwt2.allowed_audiences', 'es01,es02,es03'
+  setting 'xpack.security.authc.realms.jwt.jwt2.allowed_signature_algorithms', 'HS256,HS384'
+  setting 'xpack.security.authc.realms.jwt.jwt2.claims.principal', 'email'
+  setting 'xpack.security.authc.realms.jwt.jwt2.claim_patterns.principal', '^(.*)@[^.]*[.]example[.]com$'
+  setting 'xpack.security.authc.realms.jwt.jwt2.authorization_realms', 'lookup_native'
+  setting 'xpack.security.authc.realms.jwt.jwt2.client_authentication.type', 'SharedSecret'
+  keystore 'xpack.security.authc.realms.jwt.jwt2.client_authentication.shared_secret', 'test-secret'
+  keystore 'xpack.security.authc.realms.jwt.jwt2.hmac_key', 'test-HMAC/secret passphrase-value'
+
+  setting 'xpack.security.authc.realms.jwt.jwt3.order', '4'
+  setting 'xpack.security.authc.realms.jwt.jwt3.allowed_issuer', 'jwt3-issuer'
+  setting 'xpack.security.authc.realms.jwt.jwt3.allowed_audiences', '[jwt3-audience]'
+  setting 'xpack.security.authc.realms.jwt.jwt3.allowed_signature_algorithms', '[HS384, HS512]'
+  setting 'xpack.security.authc.realms.jwt.jwt3.claims.principal', 'sub'
+  setting 'xpack.security.authc.realms.jwt.jwt3.client_authentication.type', 'SharedSecret'
+  keystore 'xpack.security.authc.realms.jwt.jwt3.hmac_jwkset',
+   '{"keys":[{"kty":"oct","kid":"test-hmac-384","k":"W3mR8v_MP0_YdDo1OB0uwOgPX6-7PzkICVxMDVCZlPGw3vyPr8SRb5akrRSNU-zV"},{"kty":"oct","kid":"test-hmac-512","k":"U4kMAa7tBwKOD4ggab4ZRGeHlFTILgNbescS1b5nambKJPmrB7QjeTryvfrE8zjYSvLxW2-tzFJUpk38a6FjPA"}]}'
+  keystore 'xpack.security.authc.realms.jwt.jwt3.client_authentication.shared_secret', 'test-secret'
+
+
+  user username: "admin_user", password: "admin-password"
+}
+