feat: Retry options implementation across all APIs (#10183)

* Custom Retry options changes in all APIs and RetryHandler changes
googleapis · Apr 19, 2023 · cc384b5 · cc384b5
1 parent 6e52ed9
commit cc384b5
Show file tree

Hide file tree

Showing 43 changed files with 189 additions and 61 deletions.
diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateBucketOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateBucketOptions.cs
@@ -38,6 +38,11 @@ public sealed class CreateBucketOptions
         /// </summary>
         public Projection? Projection { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(BucketsResource.InsertRequest request)
         {
             if (PredefinedAcl != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateHmacKeyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateHmacKeyOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC
+// Copyright 2019 Google LLC
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@ namespace Google.Cloud.Storage.V1
     /// </summary>
     public sealed class CreateHmacKeyOptions
     {
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(HmacKeysResource.CreateRequest request)
         {
         }

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateNotificationOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/CreateNotificationOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class CreateNotificationOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(NotificationsResource.InsertRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteBucketOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteBucketOptions.cs
@@ -63,6 +63,11 @@ public sealed class DeleteBucketOptions
         /// </remarks>
         public bool? DeleteObjects { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(BucketsResource.DeleteRequest request)
         {
             if (IfMetagenerationMatch != null && IfMetagenerationNotMatch != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteHmacKeyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteHmacKeyOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC
+// Copyright 2019 Google LLC
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@ namespace Google.Cloud.Storage.V1
     /// </summary>
     public sealed class DeleteHmacKeyOptions
     {
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(ProjectsResource.HmacKeysResource.DeleteRequest request)
         {
         }

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteNotificationOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteNotificationOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class DeleteNotificationOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(NotificationsResource.DeleteRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteObjectOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/DeleteObjectOptions.cs
@@ -58,6 +58,11 @@ public sealed class DeleteObjectOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(ObjectsResource.DeleteRequest request)
         {
             // Note the use of ArgumentException here, as this will basically be the result of invalid
@@ -74,12 +79,10 @@ internal void ModifyRequest(ObjectsResource.DeleteRequest request)
             if (Generation != null)
             {
                 request.Generation = Generation;
-                RetryHandler.MarkAsRetriable(request);
             }
             if (IfGenerationMatch != null)
             {
                 request.IfGenerationMatch = IfGenerationMatch;
-                RetryHandler.MarkAsRetriable(request);
             }
             if (IfGenerationNotMatch != null)
             {

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetBucketIamPolicyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetBucketIamPolicyOptions.cs
@@ -35,6 +35,11 @@ public sealed class GetBucketIamPolicyOptions
         /// </summary>
         public int? RequestedPolicyVersion { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(GetIamPolicyRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetBucketOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetBucketOptions.cs
@@ -48,7 +48,7 @@ public sealed class GetBucketOptions
         public string UserProject { get; set; }
 
         /// <summary>
-        /// Options to pass custom retry configuration for each API request
+        /// Options to pass custom retry configuration for each API request.
         /// </summary>
         public RetryOptions RetryOptions { get; set; }  
 

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetHmacKeyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetHmacKeyOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC
+// Copyright 2019 Google LLC
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,11 @@ namespace Google.Cloud.Storage.V1
     /// </summary>
     public sealed class GetHmacKeyOptions
     {
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(ProjectsResource.HmacKeysResource.GetRequest request)
         {
         }

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetNotificationOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetNotificationOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class GetNotificationOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(NotificationsResource.GetRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetObjectOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetObjectOptions.cs
@@ -71,6 +71,11 @@ public sealed class GetObjectOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(GetRequest request)
         {
             // Note the use of ArgumentException here, as this will basically be the result of invalid

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetStorageServiceAccountEmailOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/GetStorageServiceAccountEmailOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class GetStorageServiceAccountEmailOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(GetRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListBucketsOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListBucketsOptions.cs
@@ -57,6 +57,11 @@ public sealed class ListBucketsOptions
         /// </summary>
         public string Fields { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         /// <summary>
         /// Modifies the specified request for all non-null properties of this options object.
         /// </summary>

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListHmacKeysOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListHmacKeysOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2019 Google LLC
+// Copyright 2019 Google LLC
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -40,6 +40,11 @@ public sealed class ListHmacKeysOptions
         /// </summary>
         public bool? ShowDeletedKeys { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(ProjectsResource.HmacKeysResource.ListRequest request)
         {
             if (PageSize != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListNotificationsOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListNotificationsOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google Inc. All Rights Reserved.
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class ListNotificationsOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(ListRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListObjectsOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/ListObjectsOptions.cs
@@ -92,6 +92,11 @@ public sealed class ListObjectsOptions
         /// </summary>
         public string EndOffset { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         /// <summary>
         /// Modifies the specified request for all non-null properties of this options object.
         /// </summary>

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/LockBucketRetentionPolicyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/LockBucketRetentionPolicyOptions.cs
@@ -1,4 +1,4 @@
-// Copyright 2018 Google LLC
+// Copyright 2018 Google LLC
 // 
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -27,6 +27,11 @@ public sealed class LockBucketRetentionPolicyOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(BucketsResource.LockRetentionPolicyRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/PatchBucketOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/PatchBucketOptions.cs
@@ -57,6 +57,11 @@ public sealed class PatchBucketOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(PatchRequest request)
         {
             if (IfMetagenerationMatch != null && IfMetagenerationNotMatch != null)
@@ -67,7 +72,6 @@ internal void ModifyRequest(PatchRequest request)
             if (IfMetagenerationMatch != null)
             {
                 request.IfMetagenerationMatch = IfMetagenerationMatch;
-                RetryHandler.MarkAsRetriable(request);
             }
             if (IfMetagenerationNotMatch != null)
             {

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/PatchObjectOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/PatchObjectOptions.cs
@@ -75,6 +75,11 @@ public sealed class PatchObjectOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(PatchRequest request)
         {
             // Note the use of ArgumentException here, as this will basically be the result of invalid
@@ -103,7 +108,6 @@ internal void ModifyRequest(PatchRequest request)
             if (IfMetagenerationMatch != null)
             {
                 request.IfMetagenerationMatch = IfMetagenerationMatch;
-                RetryHandler.MarkAsRetriable(request);
             }
             if (IfMetagenerationNotMatch != null)
             {

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/RetryHandler.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/RetryHandler.cs
@@ -42,17 +42,6 @@ internal sealed class RetryHandler : IHttpUnsuccessfulResponseHandler
         private RetryHandler(RetryOptions retryOptions, IScheduler scheduler) =>
             (_retryOptions, _scheduler) = (retryOptions, scheduler);
 
-        // TODO: Will remove it once the implementation across all APIs is complete
-        internal static void MarkAsRetriable<TResponse>(StorageBaseServiceRequest<TResponse> request)
-        {
-            RetryHandler retryHandler = new RetryHandler(RetryOptions.IdempotentRetryOptions, SystemScheduler.Instance);
-
-            // Note: we can't use ModifyRequest, as the x-goog-api-client header is added later by ConfigurableMessageHandler.
-            // Additionally, that's only called once, and we may want to record the attempt number as well.
-            request.AddExecuteInterceptor(InvocationIdInterceptor.Instance);
-            request.AddUnsuccessfulResponseHandler(retryHandler);
-        }
-
         /// <summary>
         /// It marks the request as retriable with the retry options as provided.
         /// In case null retry options or Never is passed, retry will not happen in case of failure.

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/SetBucketIamPolicyOptions.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/SetBucketIamPolicyOptions.cs
@@ -27,6 +27,11 @@ public sealed class SetBucketIamPolicyOptions
         /// </summary>
         public string UserProject { get; set; }
 
+        /// <summary>
+        /// Options to pass custom retry configuration for each API request.
+        /// </summary>
+        public RetryOptions RetryOptions { get; set; }
+
         internal void ModifyRequest(SetIamPolicyRequest request)
         {
             if (UserProject != null)

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CopyObject.cs
@@ -74,10 +74,9 @@ public sealed partial class StorageClientImpl : StorageClient
             Object obj = options?.ExtraMetadata ?? new Object();
             var request = Service.Objects.Rewrite(obj, sourceBucket, sourceObjectName, destinationBucket, destinationObjectName);
 
+            options?.ModifyRequest(request);
             RetryOptions retryOptions = options?.RetryOptions ?? RetryOptions.MaybeIdempotent(options?.IfGenerationMatch);
             MarkAsRetriable(request, retryOptions);
-
-            options?.ModifyRequest(request);
             ApplyEncryptionKey(options?.EncryptionKey, options?.KmsKeyName, request);
             request.ModifyRequest += (options?.SourceEncryptionKey ?? EncryptionKey).ModifyRequestForRewriteSource;
             return request;

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CreateBucket.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.CreateBucket.cs
@@ -46,7 +46,8 @@ private BucketsResource.InsertRequest CreateCreateBucketRequest(string projectId
             var bucketEntity = new Bucket { Name = bucket };
             var request = Service.Buckets.Insert(bucketEntity, projectId);
             options?.ModifyRequest(request);
-            RetryHandler.MarkAsRetriable(request);
+            RetryOptions retryOptions = options?.RetryOptions ?? RetryOptions.IdempotentRetryOptions;
+            MarkAsRetriable(request, retryOptions);
             return request;
         }
 
@@ -56,7 +57,8 @@ private BucketsResource.InsertRequest CreateCreateBucketRequest(string projectId
             ValidateBucket(bucket, nameof(bucket));
             var request = Service.Buckets.Insert(bucket, projectId);
             options?.ModifyRequest(request);
-            RetryHandler.MarkAsRetriable(request);
+            RetryOptions retryOptions = options?.RetryOptions ?? RetryOptions.IdempotentRetryOptions;
+            MarkAsRetriable(request, retryOptions);
             return request;
         }
     }

diff --git a/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DeleteBucket.cs b/apis/Google.Cloud.Storage.V1/Google.Cloud.Storage.V1/StorageClientImpl.DeleteBucket.cs
@@ -104,7 +104,8 @@ private BucketsResource.DeleteRequest CreateDeleteBucketRequest(string bucket, D
         {
             var request = Service.Buckets.Delete(bucket);
             options?.ModifyRequest(request);
-            RetryHandler.MarkAsRetriable(request);
+            RetryOptions retryOptions = options?.RetryOptions ?? RetryOptions.IdempotentRetryOptions;
+            MarkAsRetriable(request, retryOptions);
             return request;
         }
     }