feat: Add configurable retry timing for RunTransactionAsync

There are further changes to come in terms of transaction retry, but
this is at least a start. Note that this changes the default backoff
from "none at all" to "100ms initial, with a multiplier of 1.3".
That's a more reasonable default, and it seems unlikely that
customers would actually *depend* on there being no backoff.

Fixes #10513

apis/Google.Cloud.Firestore/Google.Cloud.Firestore.Tests/FirestoreDbTest.RunTransactionAsync.cs

@@ -1,4 +1,4 @@
-// Copyright 2017, Google Inc. All rights reserved.
+// Copyright 2017, Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,11 +12,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+using Google.Api.Gax;
+using Google.Api.Gax.Grpc;
+using Google.Api.Gax.Testing;
 using Google.Cloud.Firestore.V1;
 using Google.Protobuf;
 using Grpc.Core;
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Threading.Tasks;
 using Xunit;
 using static Google.Cloud.Firestore.Tests.ProtoHelpers;
@@ -138,6 +143,35 @@ public async Task RunTransactionAsync_TooManyRetries(int? userSpecifiedAttempts)
             Assert.Equal(actualAttempts, client.RollbackRequests.Count);
         }
 
+        [Fact]
+        public async Task RunTransactionAsync_CustomRetrySettings()
+        {
+            var start = new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+            var clock = new FakeClock(start);
+            var scheduler = new FakeScheduler(clock);
+
+            var settings = new FirestoreSettings
+            {
+                Scheduler = scheduler,
+                Clock = scheduler.Clock
+            };
+
+            // 6 failures, so 7 RPCs in total.
+            var client = new TransactionTestingClient(6, CreateRpcException(StatusCode.Aborted), settings);
+            var db = FirestoreDb.Create("proj", "db", client);
+
+            // Backoffs will be 1, 2, 4, 5, 5, 5.
+            // Timestamps will be 0, 1, 3, 7, 12, 17, 22.
+            var retrySettings = RetrySettings.FromExponentialBackoff(10, TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5), 2.0, ex => true, RetrySettings.NoJitter);
+            var options = TransactionOptions.ForRetrySettings(retrySettings);
+
+            var timestamps = await scheduler.RunAsync(() => db.RunTransactionAsync(CreateTimestampingCallback(scheduler.Clock), options));
+
+            var timestampSecondsSinceStart = timestamps.Select(ts => (ts - start).TotalSeconds).ToArray();
+            double[] expectedSecondsSinceStart = { 0.0, 1.0, 3.0, 7.0, 12.0, 17.0, 22.0 };
+            Assert.Equal(expectedSecondsSinceStart, timestampSecondsSinceStart);
+        }
+
         /// <summary>
         /// Creates a request that creates projects/proj/databases/db/documents/col/doc1 and
         /// deletes projects/proj/databases/db/documents/col/doc2 - the operations performed in
@@ -184,6 +218,19 @@ public async Task RunTransactionAsync_TooManyRetries(int? userSpecifiedAttempts)
             };
         }
 
+        private Func<Transaction, Task<List<DateTime>>> CreateTimestampingCallback(IClock clock)
+        {
+            List<DateTime> ret = new();
+            return async transaction =>
+            {
+                var db = transaction.Database;
+                ret.Add(clock.GetCurrentDateTimeUtc());
+                await transaction.GetSnapshotAsync(db.Document("col/x"));
+                transaction.Create(db.Document("col/doc1"), new { Name = "Test" });
+                transaction.Delete(db.Document("col/doc2"));
+                return ret;
+            };
+        }
 
         private static RollbackRequest CreateRollbackRequest(string transactionId) =>
             new RollbackRequest

apis/Google.Cloud.Firestore/Google.Cloud.Firestore.Tests/TransactionTestingClient.cs

@@ -1,4 +1,4 @@
-// Copyright 2017, Google Inc. All rights reserved.
+// Copyright 2017, Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -57,9 +57,9 @@ public TransactionTestingClient() : this(0, null)
         /// <summary>
         /// Creates a client which will respond to the first <paramref name="failures"/> commit calls with <paramref name="exception"/>.
         /// </summary>
-        public TransactionTestingClient(int failures, Exception exception)
+        public TransactionTestingClient(int failures, Exception exception, FirestoreSettings settings = null)
         {
-            Settings = FirestoreSettings.GetDefault();
+            Settings = settings ?? FirestoreSettings.GetDefault();
             _failures = failures;
             _exception = exception;
         }

apis/Google.Cloud.Firestore/Google.Cloud.Firestore/FirestoreDb.cs

@@ -387,9 +387,12 @@ public Task RunTransactionAsync(Func<Transaction, Task> callback, TransactionOpt
         public async Task<T> RunTransactionAsync<T>(Func<Transaction, Task<T>> callback, TransactionOptions options = null, CancellationToken cancellationToken = default)
         {
             ByteString previousTransactionId = null;
-            options = options ?? TransactionOptions.Default;
-            var attemptsLeft = options.MaxAttempts;
-            TimeSpan backoff = TimeSpan.FromSeconds(1);
+            options ??= TransactionOptions.Default;
+
+            var retrySettings = options.RetrySettings;
+            var attemptsLeft = retrySettings.MaxAttempts;
+            TimeSpan backoff = retrySettings.InitialBackoff;
+            var scheduler = Client.Settings.Scheduler ?? SystemScheduler.Instance;
 
             while (true)
             {
@@ -408,8 +411,12 @@ public async Task<T> RunTransactionAsync<T>(Func<Transaction, Task<T>> callback,
                     }
                     catch (RpcException e) when (CheckRetry(e, ref rollback))
                     {
-                        // On to the next iteration...
+                        // On to the next iteration after a backoff.
                     }
+
+                    // This is essentially the inner loop of RetryAttempt.CreateRetrySequence.
+                    await scheduler.Delay(retrySettings.BackoffJitter.GetDelay(backoff), cancellationToken).ConfigureAwait(false);
+                    backoff = retrySettings.NextBackoff(backoff);
                 }
                 finally
                 {

apis/Google.Cloud.Firestore/Google.Cloud.Firestore/TransactionOptions.cs

@@ -1,4 +1,4 @@
-// Copyright 2017, Google Inc. All rights reserved.
+// Copyright 2017, Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,6 +13,8 @@
 // limitations under the License.
 
 using Google.Api.Gax;
+using Google.Api.Gax.Grpc;
+using System;
 
 namespace Google.Cloud.Firestore
 {
@@ -24,27 +26,46 @@ public sealed class TransactionOptions
         /// <summary>
         /// The transaction options that are used if nothing is specified by the caller.
         /// </summary>
-        public static TransactionOptions Default { get; } = new TransactionOptions(5);
+        public static TransactionOptions Default { get; } = ForMaxAttempts(5);
 
         /// <summary>
         /// The number of times the transaction will be attempted before failing.
+        /// This is equivalent to <see cref="RetrySettings.MaxAttempts"/>.
         /// </summary>
-        public int MaxAttempts { get; }
+        public int MaxAttempts => RetrySettings.MaxAttempts;
 
-        private TransactionOptions(int maxAttempts)
+        /// <summary>
+        /// The settings to control the timing of retries within the transaction.
+        /// The <see cref="RetrySettings.RetryFilter"/> property is ignored. This property is never null.
+        /// </summary>
+        public RetrySettings RetrySettings { get; }
+
+        private TransactionOptions(RetrySettings retrySettings)
         {
-            MaxAttempts = maxAttempts;
+            RetrySettings = retrySettings;
         }
 
         /// <summary>
-        /// Creates an instance with the given maximum number of attempts.
+        /// Creates an instance with the given maximum number of attempts. The default retry
+        /// timing will be used.
         /// </summary>
         /// <param name="maxAttempts">The number of times a transaction will be attempted before failing. Must be positive.</param>
         /// <returns>A new options object.</returns>
-        public static TransactionOptions ForMaxAttempts(int maxAttempts)
-        {
-            GaxPreconditions.CheckArgumentRange(maxAttempts, nameof(maxAttempts), 1, int.MaxValue);
-            return new TransactionOptions(maxAttempts);
-        }
+        public static TransactionOptions ForMaxAttempts(int maxAttempts) =>
+            new TransactionOptions(RetrySettings.FromExponentialBackoff(
+                maxAttempts: maxAttempts,
+                initialBackoff: TimeSpan.FromMilliseconds(100),
+                maxBackoff: TimeSpan.FromMinutes(1),
+                backoffMultiplier: 1.3,
+                retryFilter: x => true)); // Ignored
+
+        /// <summary>
+        /// Creates an instance with the given retry settings, including maximum number of attempts.
+        /// The <see cref="RetrySettings.RetryFilter"/> property is not used.
+        /// </summary>
+        /// <param name="retrySettings">The retry settings to use. Must not be null.</param>
+        /// <returns>A new options object.</returns>
+        public static TransactionOptions ForRetrySettings(RetrySettings retrySettings) =>
+            new TransactionOptions(GaxPreconditions.CheckNotNull(retrySettings, nameof(retrySettings)));
     }
 }