feat: Add operator access to configure AWS retry policy. Default to STANDARD, recommended by AWS

cloud/aws-common/pom.xml

@@ -82,6 +82,17 @@
           <artifactId>sdk-core</artifactId>
           <version>${aws.sdk.v2.version}</version>
         </dependency>
+        <!-- Retry strategies configured by AWSClientConfig -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>retries-spi</artifactId>
+            <version>${aws.sdk.v2.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>retries</artifactId>
+            <version>${aws.sdk.v2.version}</version>
+        </dependency>
         <!-- WebIdentityTokenProvider requires runtime dependency on sts -->
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
@@ -115,5 +126,10 @@
             <artifactId>junit-jupiter-api</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.junit.jupiter</groupId>
+            <artifactId>junit-jupiter-params</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 </project>

cloud/aws-common/src/main/java/org/apache/druid/common/aws/AWSClientConfig.java

@@ -20,10 +20,18 @@
 package org.apache.druid.common.aws;
 
 import com.fasterxml.jackson.annotation.JacksonInject;
+import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.StringUtils;
 import org.apache.druid.utils.RuntimeInfo;
+import software.amazon.awssdk.awscore.retry.AwsRetryStrategy;
+import software.amazon.awssdk.retries.api.RetryStrategy;
 
 import javax.annotation.Nullable;
+import javax.validation.constraints.Min;
+import java.util.Arrays;
 
 public class AWSClientConfig
 {
@@ -36,6 +44,61 @@ public class AWSClientConfig
   /** AWS SDK v2's own default. */
   private static final int DEFAULT_MAX_CONNECTIONS_FLOOR = 50;
 
+  /**
+   * Retry strategy family. Declared as an enum so an unrecognised value is rejected while the config is bound at
+   * startup, rather than when a client is first built.
+   */
+  public enum RetryMode
+  {
+    STANDARD {
+      @Override
+      RetryStrategy createStrategy()
+      {
+        // Pass true to ensure we get the new standard AWS SDKv2 retry behavior and not legacy behavior.
+        return AwsRetryStrategy.standardRetryStrategy(true);
+      }
+    },
+    ADAPTIVE {
+      @Override
+      RetryStrategy createStrategy()
+      {
+        // Standard plus a client-side rate limiter, which unlike standard can delay or block the initial request,
+        // not just retries. The limiter belongs to one client instance and covers every request that client makes,
+        // so throttling on one key prefix also slows requests to prefixes that are not being throttled.
+        return AwsRetryStrategy.adaptiveRetryStrategy(true);
+      }
+    },
+    LEGACY {
+      @Override
+      RetryStrategy createStrategy()
+      {
+        // Deliberately left on the pre-standard behavior: this mode exists so a deployment can get back to what it
+        // had before, which is the opposite of what the opt-in above asks for.
+        return AwsRetryStrategy.legacyRetryStrategy();
+      }
+    };
+
+    abstract RetryStrategy createStrategy();
+
+    @JsonValue
+    @Override
+    public String toString()
+    {
+      return StringUtils.toLowerCase(name());
+    }
+
+    @JsonCreator
+    public static RetryMode fromString(String value)
+    {
+      for (RetryMode mode : values()) {
+        if (mode.name().equalsIgnoreCase(value)) {
+          return mode;
+        }
+      }
+      throw new IAE("Invalid druid.s3.retryMode[%s]. Must be one of %s.", value, Arrays.toString(values()));
+    }
+  }
+
   /**
    * Used by {@link #getMaxConnections} to scale the default connection pool with host size so hosts large enough to
    * do a lot of concurrent deep-storage I/O (e.g. virtual-storage historicals fanning out on-demand loads to S3)
@@ -80,6 +143,26 @@ public class AWSClientConfig
   @Nullable
   private Integer maxConnections = null;
 
+  /**
+   * Retry strategy applied to every AWS client built from this config.
+   */
+  @JsonProperty
+  private RetryMode retryMode = RetryMode.STANDARD;
+
+  /**
+   * Total attempts per request, including the first. A value of 1 disables
+   * retries. Null leaves the count that {@link #retryMode} defines for itself, which AWS tunes alongside that mode's
+   * backoff and retry quota.
+   * <p>
+   * This counts HTTP requests. Druid layers its own retries on top (see {@code S3Utils#retryS3Operation}) and the two
+   * multiply, but they are not equivalent: an attempt here re-sends a single request, whereas a Druid-level retry
+   * repeats a whole operation, such as re-uploading an entire segment.
+   */
+  @JsonProperty
+  @Nullable
+  @Min(1)
+  private Integer maxAttempts = null;
+
   public String getProtocol()
   {
     return protocol;
@@ -146,6 +229,42 @@ public int getMaxConnections()
     return Math.max(DEFAULT_MAX_CONNECTIONS_FLOOR, 4 * runtimeInfo.getAvailableProcessors());
   }
 
+  public RetryMode getRetryMode()
+  {
+    return retryMode;
+  }
+
+  @Nullable
+  public Integer getMaxAttempts()
+  {
+    return maxAttempts;
+  }
+
+  /**
+   * Builds the strategy to hand to {@code ClientOverrideConfiguration.retryStrategy}. Kept as a plain function of the
+   * config because a built AWS client does not expose the strategy it was given, so this is the only place the
+   * mapping can be tested.
+   * <p>
+   * Returns a new instance per call; clients must not share one, since the strategies hold their circuit-breaker
+   * quota (and, for adaptive, their rate limiter) on the instance.
+   */
+  public RetryStrategy getRetryStrategy()
+  {
+    return withMaxAttempts(retryMode.createStrategy());
+  }
+
+  /**
+   * Overrides the attempt count only when one is configured, so an unset {@link #maxAttempts} leaves whatever
+   * the chosen mode defines for itself.
+   */
+  private RetryStrategy withMaxAttempts(RetryStrategy strategy)
+  {
+    if (maxAttempts == null) {
+      return strategy;
+    }
+    return strategy.toBuilder().maxAttempts(maxAttempts).build();
+  }
+
   @Override
   public String toString()
   {
@@ -157,6 +276,8 @@ public String toString()
            ", connectionTimeout=" + connectionTimeout +
            ", socketTimeout=" + socketTimeout +
            ", maxConnections=" + getMaxConnections() +
+           ", retryMode='" + retryMode + '\'' +
+           ", maxRetryAttempts=" + maxAttempts +
            '}';
   }
 }