[s3-repository] Support IAM roles for Kubernetes service accounts (#8…

…1255)

There have been many requests to support repository-s3 authentication via IAM roles in Kubernetes service accounts.

The AWS SDK is supposed to support them out of the box with the aws-java-sdk-sts library. Unfortunately, we can't use WebIdentityTokenCredentialsProvider from the SDK. It reads the token from AWS_WEB_IDENTITY_TOKEN_FILE environment variable which is usually mounted to /var/run/secrets/eks.amazonaws.com/serviceaccount/token and the S3 repository doesn't have the read permission to read it. We don't want to hard-code a file permission for the repository, because the location of AWS_WEB_IDENTITY_TOKEN_FILE can change at any time in the future and we would also generally prefer to restrict the ability of plugins to access things outside of their config directory.

To overcome this limitation, this change adds a custom WebIdentityCredentials provider that reads the service account from a symlink to AWS_WEB_IDENTITY_TOKEN_FILE created in the repository's config directory. We expect the end user to create the symlink to indicate that they want to use service accounts for authentification.

Service accounts are checked and exchanged for session tokens by the AWS STS. To test the authentification flow, this change adds a test fixture which mocks the assume-role-with-web-identity call to the service and returns a response with test credentials.

Fixes #52625

docs/changelog/81255.yaml

@@ -0,0 +1,6 @@
+pr: 81255
+summary: "[s3-repository] Support IAM roles for Kubernetes service accounts"
+area: Snapshot/Restore
+type: feature
+issues:
+ - 52625

docs/reference/snapshot-restore/repository-s3.asciidoc

@@ -12,10 +12,9 @@ https://www.elastic.co/cloud/.*
 To register an S3 repository, specify the type as `s3` when creating
 the repository. The repository defaults to using
 https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html[ECS
-IAM Role] or
-https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html[EC2
-IAM Role] credentials for authentication. The only mandatory setting is the
-bucket name:
+IAM Role] credentials for authentication. You can also use <<iam-kubernetes-service-accounts>> Kubernetes service accounts.
+
+The only mandatory setting is the bucket name:
 
 [source,console]
 ----
@@ -495,3 +494,24 @@ bandwidth of your VPC's NAT instance.
 
 Instances residing in a public subnet in an AWS VPC will connect to S3 via the
 VPC's internet gateway and not be bandwidth limited by the VPC's NAT instance.
+
+
+[[iam-kubernetes-service-accounts]]
+[discrete]
+==== Using IAM roles for Kubernetes service accounts for authentication
+If you want to use https://aws.amazon.com/blogs/opensource/introducing-fine-grained-iam-roles-service-accounts/[Kubernetes service accounts]
+for authentication, you need to add a symlink to the `$AWS_WEB_IDENTITY_TOKEN_FILE` environment variable
+(which should be automatically set by a Kubernetes pod) in the S3 repository config directory, so the repository
+can have the read access for the service account (a repository can't read any files outside its config directory).
+For example:
+
+[source,bash]
+----
+mkdir -p "${ES_PATH_CONF}/repository-s3"
+ln -s $AWS_WEB_IDENTITY_TOKEN_FILE "${ES_PATH_CONF}/repository-s3/aws-web-identity-token-file"
+----
+
+IMPORTANT: The symlink must be created on all data and master eligible nodes and be readable
+by the `elasticsearch` user. By default, {es} runs as user `elasticsearch` using uid:gid `1000:0`.
+
+If the symlink exists, it will be used by default by all S3 repositories that don't have explicit `client` credentials.

modules/repository-s3/build.gradle

@@ -29,6 +29,7 @@ versions << [
 dependencies {
   api "com.amazonaws:aws-java-sdk-s3:${versions.aws}"
   api "com.amazonaws:aws-java-sdk-core:${versions.aws}"
+  api "com.amazonaws:aws-java-sdk-sts:${versions.aws}"
   api "com.amazonaws:jmespath-java:${versions.aws}"
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
@@ -105,6 +106,9 @@ String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
 String s3ECSBucket = System.getenv("amazon_s3_bucket_ecs")
 String s3ECSBasePath = System.getenv("amazon_s3_base_path_ecs")
 
+String s3STSBucket = System.getenv("amazon_s3_bucket_sts")
+String s3STSBasePath = System.getenv("amazon_s3_base_path_sts")
+
 boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
 
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
@@ -143,6 +147,13 @@ if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
   throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
 }
 
+if (!s3STSBucket && !s3STSBasePath) {
+  s3STSBucket = 'sts_bucket'
+  s3STSBasePath = 'sts_base_path'
+} else if (!s3STSBucket || !s3STSBasePath) {
+  throw new IllegalArgumentException("not all options specified to run STS tests are present")
+}
+
 tasks.named("processYamlRestTestResources").configure {
   Map<String, Object> expansions = [
     'permanent_bucket'        : s3PermanentBucket,
@@ -153,6 +164,8 @@ tasks.named("processYamlRestTestResources").configure {
     'ec2_base_path'           : s3EC2BasePath,
     'ecs_bucket'              : s3ECSBucket,
     'ecs_base_path'           : s3ECSBasePath,
+    'sts_bucket'              : s3STSBucket,
+    'sts_base_path'           : s3STSBasePath,
     'disable_chunked_encoding': s3DisableChunkedEncoding
   ]
   inputs.properties(expansions)
@@ -167,12 +180,14 @@ tasks.named("internalClusterTest").configure {
 tasks.named("yamlRestTest").configure {
     systemProperty 'tests.rest.blacklist', (
       useFixture ?
-        ['repository_s3/50_repository_ecs_credentials/*']
+        ['repository_s3/50_repository_ecs_credentials/*',
+         'repository_s3/60_repository_sts_credentials/*']
         :
         [
           'repository_s3/30_repository_temporary_credentials/*',
           'repository_s3/40_repository_ec2_credentials/*',
-          'repository_s3/50_repository_ecs_credentials/*'
+          'repository_s3/50_repository_ecs_credentials/*',
+          'repository_s3/60_repository_sts_credentials/*'
         ]
     ).join(",")
 }
@@ -226,7 +241,8 @@ if (useFixture) {
     systemProperty 'tests.rest.blacklist', [
       'repository_s3/30_repository_temporary_credentials/*',
       'repository_s3/40_repository_ec2_credentials/*',
-      'repository_s3/50_repository_ecs_credentials/*'
+      'repository_s3/50_repository_ecs_credentials/*',
+      'repository_s3/60_repository_sts_credentials/*'
     ].join(",")
   }
   tasks.named("check").configure { dependsOn("yamlRestTestMinio") }
@@ -253,7 +269,8 @@ if (useFixture) {
       'repository_s3/10_basic/*',
       'repository_s3/20_repository_permanent_credentials/*',
       'repository_s3/30_repository_temporary_credentials/*',
-      'repository_s3/40_repository_ec2_credentials/*'
+      'repository_s3/40_repository_ec2_credentials/*',
+      'repository_s3/60_repository_sts_credentials/*'
     ].join(",")
   }
   tasks.named("check").configure { dependsOn("yamlRestTestECS") }
@@ -265,6 +282,46 @@ if (useFixture) {
   }
 }
 
+// STS (Secure Token Service)
+if (useFixture) {
+  testFixtures.useFixture(':test:fixtures:s3-fixture', 's3-fixture-with-sts')
+  tasks.register("yamlRestTestSTS", RestIntegTestTask.class) {
+    description = "Runs tests with the STS (Secure Token Service)"
+    dependsOn('bundlePlugin')
+
+    SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
+    SourceSet yamlRestTestSourceSet = sourceSets.getByName(InternalYamlRestTestPlugin.SOURCE_SET_NAME)
+    setTestClassesDirs(yamlRestTestSourceSet.getOutput().getClassesDirs())
+    setClasspath(yamlRestTestSourceSet.getRuntimeClasspath())
+    systemProperty 'tests.rest.blacklist', [
+      'repository_s3/10_basic/*',
+      'repository_s3/20_repository_permanent_credentials/*',
+      'repository_s3/30_repository_temporary_credentials/*',
+      'repository_s3/40_repository_ec2_credentials/*',
+      'repository_s3/50_repository_ecs_credentials/*'
+    ].join(",")
+  }
+  tasks.named("check").configure { dependsOn("yamlRestTestSTS") }
+
+  testClusters.matching { it.name == "yamlRestTestSTS" }.configureEach {
+    module tasks.bundlePlugin.archiveFile
+
+    setting 's3.client.integration_test_sts.endpoint', { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-sts', '80')}" }, IGNORE_VALUE
+    systemProperty 'com.amazonaws.sdk.stsMetadataServiceEndpointOverride',
+      { "${-> fixtureAddress('s3-fixture', 's3-fixture-with-sts', '80')}/assume-role-with-web-identity" }, IGNORE_VALUE
+
+    File awsWebIdentityTokenExternalLocation = file('src/test/resources/aws-web-identity-token-file')
+    // The web identity token can be read only from the plugin config directory because of security restrictions
+    // Ideally we would create a symlink, but extraConfigFile doesn't support it
+    extraConfigFile 'repository-s3/aws-web-identity-token-file', awsWebIdentityTokenExternalLocation
+    environment 'AWS_WEB_IDENTITY_TOKEN_FILE', "$awsWebIdentityTokenExternalLocation"
+
+    // The AWS STS SDK requires the role and session names to be set. We can verify that they are sent to S3S in the S3HttpFixtureWithSTS fixture
+    environment 'AWS_ROLE_ARN', 'arn:aws:iam::123456789012:role/FederatedWebIdentityRole'
+    environment 'AWS_ROLE_SESSION_NAME', 'sts-fixture-test'
+  }
+}
+
 // 3rd Party Tests
 TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
   SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);

modules/repository-s3/licenses/aws-java-sdk-sts-1.11.749.jar.sha1

@@ -0,0 +1 @@
+724bd22c0ff41c496469e18f9bea12bdfb2f7540

modules/repository-s3/src/internalClusterTest/java/org/elasticsearch/repositories/s3/S3BlobStoreRepositoryTests.java

@@ -226,7 +226,7 @@ protected S3Repository createRepository(
             BigArrays bigArrays,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, bigArrays, recoverySettings) {
+            return new S3Repository(metadata, registry, getService(), clusterService, bigArrays, recoverySettings) {
 
                 @Override
                 public BlobStore blobStore() {

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3RepositoryPlugin.java

@@ -10,28 +10,38 @@
 
 import com.amazonaws.util.json.Jackson;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.SpecialPermission;
+import org.elasticsearch.client.internal.Client;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.RepositoryMetadata;
 import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.BigArrays;
 import org.elasticsearch.env.Environment;
+import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.indices.recovery.RecoverySettings;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.plugins.ReloadablePlugin;
 import org.elasticsearch.plugins.RepositoryPlugin;
+import org.elasticsearch.repositories.RepositoriesService;
 import org.elasticsearch.repositories.Repository;
+import org.elasticsearch.script.ScriptService;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.watcher.ResourceWatcherService;
 import org.elasticsearch.xcontent.NamedXContentRegistry;
 
 import java.io.IOException;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
+import java.util.function.Supplier;
 
 /**
  * A plugin to add a repository type that writes to and from the AWS S3.
@@ -54,17 +64,15 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
         });
     }
 
-    protected final S3Service service;
+    private final SetOnce<S3Service> service = new SetOnce<>();
+    private final Settings settings;
 
-    public S3RepositoryPlugin(final Settings settings) {
-        this(settings, new S3Service());
+    public S3RepositoryPlugin(Settings settings) {
+        this.settings = settings;
     }
 
-    S3RepositoryPlugin(final Settings settings, final S3Service service) {
-        this.service = Objects.requireNonNull(service, "S3 service must not be null");
-        // eagerly load client settings so that secure settings are read
-        final Map<String, S3ClientSettings> clientsSettings = S3ClientSettings.load(settings);
-        this.service.refreshAndClearCache(clientsSettings);
+    S3Service getService() {
+        return service.get();
     }
 
     // proxy method for testing
@@ -75,7 +83,30 @@ protected S3Repository createRepository(
         final BigArrays bigArrays,
         final RecoverySettings recoverySettings
     ) {
-        return new S3Repository(metadata, registry, service, clusterService, bigArrays, recoverySettings);
+        return new S3Repository(metadata, registry, service.get(), clusterService, bigArrays, recoverySettings);
+    }
+
+    @Override
+    public Collection<Object> createComponents(
+        Client client,
+        ClusterService clusterService,
+        ThreadPool threadPool,
+        ResourceWatcherService resourceWatcherService,
+        ScriptService scriptService,
+        NamedXContentRegistry xContentRegistry,
+        Environment environment,
+        NodeEnvironment nodeEnvironment,
+        NamedWriteableRegistry namedWriteableRegistry,
+        IndexNameExpressionResolver indexNameExpressionResolver,
+        Supplier<RepositoriesService> repositoriesServiceSupplier
+    ) {
+        service.set(s3Service(environment));
+        this.service.get().refreshAndClearCache(S3ClientSettings.load(settings));
+        return List.of(service);
+    }
+
+    S3Service s3Service(Environment environment) {
+        return new S3Service(environment);
     }
 
     @Override
@@ -118,11 +149,11 @@ public List<Setting<?>> getSettings() {
     public void reload(Settings settings) {
         // secure settings should be readable
         final Map<String, S3ClientSettings> clientsSettings = S3ClientSettings.load(settings);
-        service.refreshAndClearCache(clientsSettings);
+        getService().refreshAndClearCache(clientsSettings);
     }
 
     @Override
     public void close() throws IOException {
-        service.close();
+        getService().close();
     }
 }