[8.11] Respect regional AWS STS endpoints (#101705) (#101857)

The AWS SDK supports regional STS endpoints via the AWS_STS_REGIONAL_ENDPOINTS environment variable. If the user set it to regional and provided the region in the AWS_REGION env variable, we should respect that and make the STS client use the regional adjusted STS endpoint like https://sts.us-west-2.amazonaws.com.

See https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html

Resolves #89175

docs/changelog/101705.yaml

@@ -0,0 +1,6 @@
+pr: 101705
+summary: Respect regional AWS STS endpoints
+area: Snapshot/Restore
+type: bug
+issues:
+ - 89175

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -9,6 +9,7 @@
 package org.elasticsearch.repositories.s3;
 
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.SDKGlobalConfiguration;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.AWSCredentialsProviderChain;
@@ -320,6 +321,7 @@ static class CustomWebIdentityTokenCredentialsProvider implements AWSCredentials
 
         private STSAssumeRoleWithWebIdentitySessionCredentialsProvider credentialsProvider;
         private AWSSecurityTokenService stsClient;
+        private String stsRegion;
 
         CustomWebIdentityTokenCredentialsProvider(
             Environment environment,
@@ -361,10 +363,24 @@ static class CustomWebIdentityTokenCredentialsProvider implements AWSCredentials
             );
             AWSSecurityTokenServiceClientBuilder stsClientBuilder = AWSSecurityTokenServiceClient.builder();
 
-            // Custom system property used for specifying a mocked version of the STS for testing
-            String customStsEndpoint = jvmEnvironment.getProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", STS_HOSTNAME);
-            // Set the region explicitly via the endpoint URL, so the AWS SDK doesn't make any guesses internally.
-            stsClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customStsEndpoint, null));
+            // Check if we need to use regional STS endpoints
+            // https://docs.aws.amazon.com/sdkref/latest/guide/feature-sts-regionalized-endpoints.html
+            if ("regional".equalsIgnoreCase(systemEnvironment.getEnv("AWS_STS_REGIONAL_ENDPOINTS"))) {
+                // AWS_REGION should be injected by the EKS pod identity webhook:
+                // https://github.com/aws/amazon-eks-pod-identity-webhook/pull/41
+                stsRegion = systemEnvironment.getEnv(SDKGlobalConfiguration.AWS_REGION_ENV_VAR);
+                if (stsRegion != null) {
+                    stsClientBuilder.withRegion(stsRegion);
+                } else {
+                    LOGGER.warn("Unable to use regional STS endpoints because the AWS_REGION environment variable is not set");
+                }
+            }
+            if (stsRegion == null) {
+                // Custom system property used for specifying a mocked version of the STS for testing
+                String customStsEndpoint = jvmEnvironment.getProperty("com.amazonaws.sdk.stsMetadataServiceEndpointOverride", STS_HOSTNAME);
+                // Set the region explicitly via the endpoint URL, so the AWS SDK doesn't make any guesses internally.
+                stsClientBuilder.withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(customStsEndpoint, null));
+            }
             stsClientBuilder.withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()));
             stsClient = SocketAccess.doPrivileged(stsClientBuilder::build);
             try {
@@ -383,6 +399,10 @@ boolean isActive() {
             return credentialsProvider != null;
         }
 
+        String getStsRegion() {
+            return stsRegion;
+        }
+
         @Override
         public AWSCredentials getCredentials() {
             Objects.requireNonNull(credentialsProvider, "credentialsProvider is not set");

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/CustomWebIdentityTokenCredentialsProviderTests.java

@@ -22,6 +22,7 @@
 import org.junit.Assert;
 import org.mockito.Mockito;
 
+import java.io.IOException;
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.net.URLDecoder;
@@ -42,6 +43,15 @@ public class CustomWebIdentityTokenCredentialsProviderTests extends ESTestCase {
     private static final String ROLE_ARN = "arn:aws:iam::123456789012:role/FederatedWebIdentityRole";
     private static final String ROLE_NAME = "aws-sdk-java-1651084775908";
 
+    private static Environment getEnvironment() throws IOException {
+        Path configDirectory = createTempDir("web-identity-token-test");
+        Files.createDirectory(configDirectory.resolve("repository-s3"));
+        Files.writeString(configDirectory.resolve("repository-s3/aws-web-identity-token-file"), "YXdzLXdlYi1pZGVudGl0eS10b2tlbi1maWxl");
+        Environment environment = Mockito.mock(Environment.class);
+        Mockito.when(environment.configFile()).thenReturn(configDirectory);
+        return environment;
+    }
+
     @SuppressForbidden(reason = "HTTP server is used for testing")
     public void testCreateWebIdentityTokenCredentialsProvider() throws Exception {
         HttpServer httpServer = MockHttpServer.createHttp(new InetSocketAddress(InetAddress.getLoopbackAddress().getHostAddress(), 0), 0);
@@ -88,11 +98,7 @@ public void testCreateWebIdentityTokenCredentialsProvider() throws Exception {
         });
         httpServer.start();
 
-        Path configDirectory = Files.createTempDirectory("web-identity-token-test");
-        Files.createDirectory(configDirectory.resolve("repository-s3"));
-        Files.writeString(configDirectory.resolve("repository-s3/aws-web-identity-token-file"), "YXdzLXdlYi1pZGVudGl0eS10b2tlbi1maWxl");
-        Environment environment = Mockito.mock(Environment.class);
-        Mockito.when(environment.configFile()).thenReturn(configDirectory);
+        Environment environment = getEnvironment();
 
         // No region is set, but the SDK shouldn't fail because of that
         Map<String, String> environmentVariables = Map.of(
@@ -125,4 +131,32 @@ public void testCreateWebIdentityTokenCredentialsProvider() throws Exception {
             httpServer.stop(0);
         }
     }
+
+    public void testSupportRegionalizedEndpoints() throws Exception {
+        Map<String, String> environmentVariables = Map.of(
+            "AWS_WEB_IDENTITY_TOKEN_FILE",
+            "/var/run/secrets/eks.amazonaws.com/serviceaccount/token",
+            "AWS_ROLE_ARN",
+            ROLE_ARN,
+            "AWS_STS_REGIONAL_ENDPOINTS",
+            "regional",
+            "AWS_REGION",
+            "us-west-2"
+        );
+        Map<String, String> systemProperties = Map.of();
+
+        var webIdentityTokenCredentialsProvider = new S3Service.CustomWebIdentityTokenCredentialsProvider(
+            getEnvironment(),
+            environmentVariables::get,
+            systemProperties::getOrDefault,
+            Clock.systemUTC()
+        );
+        // We can't verify that webIdentityTokenCredentialsProvider's STS client uses the "https://sts.us-west-2.amazonaws.com"
+        // endpoint in a unit test. The client depends on hardcoded RegionalEndpointsOptionResolver that in turn depends
+        // on the system environment that we can't change in the test. So we just verify we that we called `withRegion`
+        // on stsClientBuilder which should internally correctly configure the endpoint when the STS client is built.
+        assertEquals("us-west-2", webIdentityTokenCredentialsProvider.getStsRegion());
+
+        webIdentityTokenCredentialsProvider.shutdown();
+    }
 }