feat: Add configurable retry timing for RunTransactionAsync

There are further changes to come in terms of transaction retry, but
this is at least a start. Note that this changes the default backoff
from "none at all" to "100ms initial, with a multiplier of 1.3".
That's a more reasonable default, and it seems unlikely that
customers would actually *depend* on there being no backoff.

Fixes googleapis#10513

apis/Google.Cloud.Firestore/Google.Cloud.Firestore/FirestoreDb.cs

@@ -387,11 +387,13 @@ public Task RunTransactionAsync(Func<Transaction, Task> callback, TransactionOpt
         /// <returns>A task which completes when the transaction has committed. The result of the task then contains the result of the callback.</returns>
         public async Task<T> RunTransactionAsync<T>(Func<Transaction, Task<T>> callback, TransactionOptions options = null, CancellationToken cancellationToken = default)
         {
-
             ByteString previousTransactionId = null;
-            options = options ?? TransactionOptions.Default;
-            var attemptsLeft = options.MaxAttempts;
-            TimeSpan backoff = TimeSpan.FromSeconds(1);
+            options ??= TransactionOptions.Default;
+
+            var retrySettings = options.RetrySettings;
+            var attemptsLeft = retrySettings.MaxAttempts;
+            TimeSpan backoff = retrySettings.InitialBackoff;
+            var scheduler = Client.Settings.Scheduler ?? SystemScheduler.Instance;
 
             while (true)
             {
@@ -410,8 +412,12 @@ public async Task<T> RunTransactionAsync<T>(Func<Transaction, Task<T>> callback,
                     }
                     catch (RpcException e) when (CheckRetry(e, ref rollback))
                     {
-                        // On to the next iteration...
+                        // On to the next iteration after a backoff.
                     }
+
+                    // This is essentially the inner loop of RetryAttempt.CreateRetrySequence.
+                    await scheduler.Delay(retrySettings.BackoffJitter.GetDelay(backoff), cancellationToken).ConfigureAwait(false);
+                    backoff = retrySettings.NextBackoff(backoff);
                 }
                 finally
                 {

apis/Google.Cloud.Firestore/Google.Cloud.Firestore/TransactionOptions.cs

@@ -1,4 +1,4 @@
-// Copyright 2017, Google Inc. All rights reserved.
+// Copyright 2017, Google Inc. All rights reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -13,6 +13,8 @@
 // limitations under the License.
 
 using Google.Api.Gax;
+using Google.Api.Gax.Grpc;
+using System;
 
 namespace Google.Cloud.Firestore
 {
@@ -24,27 +26,46 @@ public sealed class TransactionOptions
         /// <summary>
         /// The transaction options that are used if nothing is specified by the caller.
         /// </summary>
-        public static TransactionOptions Default { get; } = new TransactionOptions(5);
+        public static TransactionOptions Default { get; } = ForMaxAttempts(5);
 
         /// <summary>
         /// The number of times the transaction will be attempted before failing.
+        /// This is equivalent to <see cref="RetrySettings.MaxAttempts"/>.
         /// </summary>
-        public int MaxAttempts { get; }
+        public int MaxAttempts => RetrySettings.MaxAttempts;
 
-        private TransactionOptions(int maxAttempts)
+        /// <summary>
+        /// The settings to control the timing of retries within the transaction.
+        /// The <see cref="RetrySettings.RetryFilter"/> property is ignored. This property is never null.
+        /// </summary>
+        public RetrySettings RetrySettings { get; }
+
+        private TransactionOptions(RetrySettings retrySettings)
         {
-            MaxAttempts = maxAttempts;
+            RetrySettings = retrySettings;
         }
 
         /// <summary>
-        /// Creates an instance with the given maximum number of attempts.
+        /// Creates an instance with the given maximum number of attempts. The default retry
+        /// timing will be used.
         /// </summary>
         /// <param name="maxAttempts">The number of times a transaction will be attempted before failing. Must be positive.</param>
         /// <returns>A new options object.</returns>
-        public static TransactionOptions ForMaxAttempts(int maxAttempts)
-        {
-            GaxPreconditions.CheckArgumentRange(maxAttempts, nameof(maxAttempts), 1, int.MaxValue);
-            return new TransactionOptions(maxAttempts);
-        }
+        public static TransactionOptions ForMaxAttempts(int maxAttempts) =>
+            new TransactionOptions(RetrySettings.FromExponentialBackoff(
+                maxAttempts: maxAttempts,
+                initialBackoff: TimeSpan.FromMilliseconds(100),
+                maxBackoff: TimeSpan.FromMinutes(1),
+                backoffMultiplier: 1.3,
+                retryFilter: x => true)); // Ignored
+
+        /// <summary>
+        /// Creates an instance with the given retry settings, including maximum number of attempts.
+        /// The <see cref="RetrySettings.RetryFilter"/> property is not used.
+        /// </summary>
+        /// <param name="retrySettings">The retry settings to use. Must not be null.</param>
+        /// <returns>A new options object.</returns>
+        public static TransactionOptions ForRetrySettings(RetrySettings retrySettings) =>
+            new TransactionOptions(GaxPreconditions.CheckNotNull(retrySettings, nameof(retrySettings)));
     }
 }