Log unsuccessful attempts to get credentials from web identity tokens (…

…#88241) (#89946)

Currently, we only verify that local environment for web identity tokens is correctly set up, but we don't verify whether it's
possible to exchange the token to credentials from the STS. If we can't get credentials from the STS, we silently fall back
to the EC2 credentials provider. Let's try to log the web identity token auth errors, so the users get a clear message in the logs in case the STS is unavailable for the ES server.

docs/changelog/88241.yaml

@@ -0,0 +1,5 @@
+pr: 88241
+summary: Log unsuccessful attempts to get credentials from web identity tokens
+area: Allocation
+type: enhancement
+issues: []

modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3Service.java

@@ -39,6 +39,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Clock;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -223,7 +224,12 @@ static AWSCredentialsProvider buildCredentials(
             if (webIdentityTokenCredentialsProvider.isActive()) {
                 logger.debug("Using a custom provider chain of Web Identity Token and instance profile credentials");
                 return new PrivilegedAWSCredentialsProvider(
-                    new AWSCredentialsProviderChain(webIdentityTokenCredentialsProvider, new EC2ContainerCredentialsProviderWrapper())
+                    new AWSCredentialsProviderChain(
+                        List.of(
+                            new ErrorLoggingCredentialsProvider(webIdentityTokenCredentialsProvider, LOGGER),
+                            new ErrorLoggingCredentialsProvider(new EC2ContainerCredentialsProviderWrapper(), LOGGER)
+                        )
+                    )
                 );
             } else {
                 logger.debug("Using instance profile credentials");
@@ -375,6 +381,37 @@ public void shutdown() throws IOException {
         }
     }
 
+    static class ErrorLoggingCredentialsProvider implements AWSCredentialsProvider {
+
+        private final AWSCredentialsProvider delegate;
+        private final Logger logger;
+
+        ErrorLoggingCredentialsProvider(AWSCredentialsProvider delegate, Logger logger) {
+            this.delegate = Objects.requireNonNull(delegate);
+            this.logger = Objects.requireNonNull(logger);
+        }
+
+        @Override
+        public AWSCredentials getCredentials() {
+            try {
+                return delegate.getCredentials();
+            } catch (Exception e) {
+                logger.error(() -> "Unable to load credentials from " + delegate, e);
+                throw e;
+            }
+        }
+
+        @Override
+        public void refresh() {
+            try {
+                delegate.refresh();
+            } catch (Exception e) {
+                logger.error(() -> "Unable to refresh " + delegate, e);
+                throw e;
+            }
+        }
+    }
+
     @FunctionalInterface
     interface SystemEnvironment {
         String getEnv(String name);

modules/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AwsS3ServiceImplTests.java

@@ -17,17 +17,22 @@
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.util.Supplier;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.ESTestCase;
+import org.mockito.ArgumentCaptor;
 import org.mockito.Mockito;
 
 import java.util.Locale;
 import java.util.Map;
 
 import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.startsWith;
 
 public class AwsS3ServiceImplTests extends ESTestCase {
 
@@ -229,4 +234,40 @@ private void assertEndpoint(Settings repositorySettings, Settings settings, Stri
         assertThat(clientSettings.endpoint, is(expectedEndpoint));
     }
 
+    public void testLoggingCredentialsProviderCatchesErrors() {
+        var mockProvider = Mockito.mock(AWSCredentialsProvider.class);
+        String mockProviderErrorMessage = "mockProvider failed to generate credentials";
+        Mockito.when(mockProvider.getCredentials()).thenThrow(new IllegalStateException(mockProviderErrorMessage));
+        var mockLogger = Mockito.mock(Logger.class);
+
+        var credentialsProvider = new S3Service.ErrorLoggingCredentialsProvider(mockProvider, mockLogger);
+        var exception = expectThrows(IllegalStateException.class, credentialsProvider::getCredentials);
+        assertEquals(mockProviderErrorMessage, exception.getMessage());
+
+        var messageSupplierCaptor = ArgumentCaptor.forClass(Supplier.class);
+        var throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
+        Mockito.verify(mockLogger).error(messageSupplierCaptor.capture(), throwableCaptor.capture());
+
+        assertThat(messageSupplierCaptor.getValue().get().toString(), startsWith("Unable to load credentials from"));
+        assertThat(throwableCaptor.getValue().getMessage(), equalTo(mockProviderErrorMessage));
+    }
+
+    public void testLoggingCredentialsProviderCatchesErrorsOnRefresh() {
+        var mockProvider = Mockito.mock(AWSCredentialsProvider.class);
+        String mockProviderErrorMessage = "mockProvider failed to refresh";
+        Mockito.doThrow(new IllegalStateException(mockProviderErrorMessage)).when(mockProvider).refresh();
+        var mockLogger = Mockito.mock(Logger.class);
+
+        var credentialsProvider = new S3Service.ErrorLoggingCredentialsProvider(mockProvider, mockLogger);
+        var exception = expectThrows(IllegalStateException.class, credentialsProvider::refresh);
+        assertEquals(mockProviderErrorMessage, exception.getMessage());
+
+        var messageSupplierCaptor = ArgumentCaptor.forClass(Supplier.class);
+        var throwableCaptor = ArgumentCaptor.forClass(Throwable.class);
+        Mockito.verify(mockLogger).error(messageSupplierCaptor.capture(), throwableCaptor.capture());
+
+        assertThat(messageSupplierCaptor.getValue().get().toString(), startsWith("Unable to refresh"));
+        assertThat(throwableCaptor.getValue().getMessage(), equalTo(mockProviderErrorMessage));
+    }
+
 }