feat: Customize Retry (#10089)

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/FakeStorageService.cs

@@ -55,6 +55,14 @@ public void ExpectRequest<TResponse>(ClientServiceRequest<TResponse> request, Ht
             handler.ExpectRequest(httpRequest.RequestUri, httpRequest.Content?.ReadAsStringAsync()?.Result, responseMessage);
         }
 
+        public void ExpectRequests<TResponse>(ClientServiceRequest<TResponse> request, HttpStatusCode statusCode, int expectedCount)
+        {
+            for (int i = 0; i < expectedCount; i++)
+            {
+                ExpectRequest(request, statusCode);
+            }
+        }
+
         public void Verify() => handler.Verify();
     }
 }

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/RetryPredicateTest.cs

@@ -0,0 +1,80 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Xunit;
+
+namespace Google.Cloud.Storage.V1.Tests;
+
+/// <summary>
+/// Test the custom retry predicates variants.
+/// </summary>
+public class RetryPredicateTest
+{
+    [Fact]
+    public void NullPredicate()
+    {
+        RetryTiming retryTiming = RetryTiming.Default;
+        RetryPredicate retryPredicate = null;
+
+        RetryOptions retryOptions = new RetryOptions(retryTiming, retryPredicate);
+
+        Assert.Equal(RetryPredicate.Never, retryOptions.Predicate);
+    }
+
+    [Fact]
+    public void NullPredicateFunc()
+    {
+        RetryPredicate retryPredicate = RetryPredicate.FromErrorCodePredicate(null);
+
+        Assert.Equal(RetryPredicate.Never, retryPredicate);
+    }
+
+    [Fact]
+    public void NullErrorCodeFunc()
+    {
+        RetryPredicate retryPredicate = RetryPredicate.FromErrorCodes(null);
+
+        Assert.Equal(RetryPredicate.Never, retryPredicate);
+    }
+
+    [Fact]
+    public void NeverPredicate()
+    {
+        RetryPredicate retryPredicate = RetryPredicate.FromErrorCodes(null);
+
+        Assert.False(retryPredicate.ShouldRetry(429));
+        Assert.False(retryPredicate.ShouldRetry(502));
+        Assert.False(retryPredicate.ShouldRetry(500));
+    }
+
+    [Fact]
+    public void RetryfromErrorCodes()
+    {
+        var retryPredicate = RetryPredicate.FromErrorCodes(429, 502);
+
+        Assert.True(retryPredicate.ShouldRetry(429));
+        Assert.True(retryPredicate.ShouldRetry(502));
+        Assert.False(retryPredicate.ShouldRetry(500));
+    }
+
+    [Fact]
+    public void RetryfromErrorPredicate()
+    {
+        var retryPredicate = RetryPredicate.FromErrorCodePredicate(errorCode => errorCode >= 500);
+
+        Assert.False(retryPredicate.ShouldRetry(429));
+        Assert.True(retryPredicate.ShouldRetry(502));
+        Assert.True(retryPredicate.ShouldRetry(500));
+    }
+}

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/RetryTest.cs

@@ -0,0 +1,129 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using Google.Apis.Requests;
+using Google.Apis.Storage.v1.Data;
+using Google.Cloud.ClientTesting;
+using System;
+using System.Linq;
+using System.Net;
+using Xunit;
+
+namespace Google.Cloud.Storage.V1.Tests;
+
+/// <summary>
+/// Retries custom RetryTimings and RetryPredicate variants complete flows.
+/// </summary>
+public class RetryTest
+{
+    [Theory]
+    [InlineData(502, true, 502, 504)]
+    [InlineData(504, true, 502, 504)]
+    [InlineData(429, false, 502, 504)]
+    [InlineData(502, false)]
+    public void CustomRetryPredicateTest(int statusCode, bool success, params int[] errorCodes)
+    {
+        RetryOptions retryOptions = new RetryOptions(
+            retryTiming: new RetryTiming(initialBackoff: TimeSpan.FromSeconds(1),
+            maxBackoff: TimeSpan.FromSeconds(6), backoffMultiplier: 2),
+            retryPredicate: RetryPredicate.FromErrorCodes(errorCodes));
+
+        AssertAttempts(
+            retryOptions: retryOptions,
+            statusCode: statusCode,
+            numOfFailures: 2,
+            maximumRetries: 3,
+            success: success,
+            expectedBackOffs: new int[] { 0, 1, 3 });
+    }
+
+    [Theory]
+    [InlineData(0, 1, 2, 0, 0)]
+    [InlineData(1, 2, 2, 2, 0, 1, 3)]
+    [InlineData(1, 4, 1, 3, 0, 1, 2, 3)]
+    public void CustomRetryTimingTest(int initialBackoff, int maxBackoff, double backoffMultiplier, int numOfFailures, params int[] expectedBackOffs)
+    {
+        RetryOptions retryOptions = new RetryOptions(
+            retryTiming: new RetryTiming(initialBackoff: TimeSpan.FromSeconds(initialBackoff),
+            maxBackoff: TimeSpan.FromSeconds(maxBackoff), backoffMultiplier: backoffMultiplier),
+            retryPredicate: RetryPredicate.FromErrorCodes(502, 504));
+
+        AssertAttempts(
+            retryOptions: retryOptions,
+            statusCode: 502,
+            numOfFailures: numOfFailures,
+            maximumRetries: numOfFailures + 1,
+            success: true,
+            expectedBackOffs: expectedBackOffs);
+    }
+
+    [Fact]
+    public void ExceptionAfterRetryExhaustedTest()
+    {
+        RetryOptions retryOptions = new RetryOptions(
+            retryTiming: new RetryTiming(initialBackoff: TimeSpan.FromSeconds(1),
+            maxBackoff: TimeSpan.FromSeconds(3), backoffMultiplier: 2),
+            retryPredicate: RetryPredicate.FromErrorCodes(502, 504));
+
+        var expectedBackOffs = new[] { 0, 1, 3 };
+
+        AssertAttempts(
+           retryOptions: retryOptions,
+           statusCode: 502,
+           numOfFailures: 4,
+           maximumRetries: 3,
+           success: false,
+           expectedBackOffs: expectedBackOffs);
+    }
+
+    private static void AssertAttempts(RetryOptions retryOptions, int statusCode, int numOfFailures, int maximumRetries, bool success, params int[] expectedBackOffs)
+    {
+        var messageHandler = new ReplayingMessageHandler(VersionHeaderBuilder.HeaderName);
+        var service = new FakeStorageService(messageHandler);
+        service.HttpClient.MessageHandler.GoogleApiClientHeader = "test/fake";
+        service.HttpClient.MessageHandler.NumTries = maximumRetries;
+
+        var request = service.Buckets.Get("bucket");
+        var client = new StorageClientImpl(service);
+
+        // Retries for the the failures. Assumed that error code 504 is always included in the predicate for these tests.
+        service.ExpectRequests(request, (HttpStatusCode) 504, numOfFailures - 1);
+        if (numOfFailures > 0)
+        {
+            service.ExpectRequest(request, (HttpStatusCode) statusCode);
+        }
+
+        DateTime startTime = DateTime.UtcNow;
+        if (success)
+        {
+            // Last call is a success.
+            service.ExpectRequest(request, new Bucket());
+
+            client.GetBucket("bucket", new GetBucketOptions { RetryOptions = retryOptions });
+
+            Assert.Equal(expectedBackOffs.Count(), messageHandler.AttemptTimestamps.Count());
+            service.Verify();
+        }
+        else
+        {
+            // The call throws an exception
+            Assert.Throws<GoogleApiException>(() => client.GetBucket("bucket", new GetBucketOptions { RetryOptions = retryOptions }));
+        }
+
+        for (int i = 0; i < messageHandler.AttemptTimestamps.Count; i++)
+        {
+            Assert.Equal(expectedBackOffs[i], (messageHandler.AttemptTimestamps[i] - startTime).TotalSeconds, 0.25);
+        }
+    }
+}

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1.Tests/RetryTimingTest.cs

@@ -0,0 +1,67 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License").
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at 
+//
+// https://www.apache.org/licenses/LICENSE-2.0 
+//
+// Unless required by applicable law or agreed to in writing, software 
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and 
+// limitations under the License.
+
+using System;
+using Xunit;
+
+namespace Google.Cloud.Storage.V1.Tests;
+
+/// <summary>
+/// Test the various custom timing variants and delays.
+/// </summary>
+public class RetryTimingTest
+{
+    [Fact]
+    public void InvalidInitialBackOff() => Assert.Throws<ArgumentOutOfRangeException>(() =>
+        RetryTiming.Default.WithInitialBackoff(initialBackoff: TimeSpan.FromSeconds(-1)));
+
+    [Fact]
+    public void InvalidBackOffMultiplier() => Assert.Throws<ArgumentOutOfRangeException>(() =>
+        RetryTiming.Default.WithBackoffMultiplier(backoffMultiplier: 0));
+
+    [Fact]
+    public void InvalidMaxBackOff() => Assert.Throws<ArgumentOutOfRangeException>(() =>
+        RetryTiming.Default.WithMaxBackoff(maxBackoff: TimeSpan.FromSeconds(0)));
+
+    [Fact]
+    public void MaxBackOff_LessThanInitialBackoff() => Assert.Throws<ArgumentOutOfRangeException>(() =>
+        RetryTiming.Default.WithInitialBackoff(initialBackoff: TimeSpan.FromSeconds(4)).WithMaxBackoff(maxBackoff: TimeSpan.FromSeconds(2)));
+
+    [Theory]
+    [InlineData(1, 5, 2, 1, 2, 4, 5)]
+    [InlineData(2, 20, 3, 2, 6, 18, 20)]
+    [InlineData(0, 1, 2, 0)]
+    [InlineData(1, 2, 2, 1, 2)]
+    [InlineData(1, 4, 1, 1, 1, 1)]
+    [InlineData(8, 40, 1.5, 8, 12, 18, 27, 40, 40)]
+    public void GetDelayVariation(int initialBackoff, int maxBackoff, double backoffMultiplier, params int[] backOffs)
+    {
+        RetryTiming retryTiming = new RetryTiming(initialBackoff: TimeSpan.FromSeconds(initialBackoff), maxBackoff: TimeSpan.FromSeconds(maxBackoff), backoffMultiplier: backoffMultiplier);
+
+        for (int i = 0; i < backOffs.Length; i++)
+        {
+            Assert.Equal(TimeSpan.FromSeconds(backOffs[i]), retryTiming.GetDelay(failureCount: i + 1));
+        }
+    }
+
+    [Fact]
+    public void NullTiming_EquivalentToDefault()
+    {
+        RetryTiming retryTiming = null;
+        RetryPredicate retryPredicate = RetryPredicate.RetriableIdempotentErrors;
+        RetryOptions retryOptions = new RetryOptions(retryTiming, retryPredicate);
+
+        Assert.Equal(RetryTiming.Default, retryOptions.Timing);
+    }
+}

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CopyObjectOptions.cs

@@ -128,6 +128,11 @@ public sealed class CopyObjectOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(RewriteRequest request)
         {
             // Note the use of ArgumentException here, as this will basically be the result of invalid
@@ -166,7 +171,6 @@ internal void ModifyRequest(RewriteRequest request)
             if (IfGenerationMatch != null)
             {
                 request.IfGenerationMatch = IfGenerationMatch;
-                RetryHandler.MarkAsRetriable(request);
             }
             if (IfGenerationNotMatch != null)
             {

apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetBucketOptions.cs

@@ -47,6 +47,11 @@ public sealed class GetBucketOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }  
+
         internal void ModifyRequest(GetRequest request)
         {
             if (IfMetagenerationMatch != null && IfMetagenerationNotMatch != null)