feat: add retries to client credential requests

README.md

@@ -41,6 +41,7 @@ This is an autogenerated SDK for OpenFGA. It provides a wrapper around the [Open
     - [Assertions](#assertions)
       - [Read Assertions](#read-assertions)
       - [Write Assertions](#write-assertions)
+  - [Retries](#retries)
   - [API Endpoints](#api-endpoints)
   - [Models](#models)
 - [Contributing](#contributing)
@@ -741,6 +742,40 @@ await fgaClient.WriteAssertions(body, options);
 ```
 
 
+### Retries
+
+By default API requests are retried up to 15 times on 429 and 5xx errors and credential requests are retried up to 15 times on 429 and 5xx errors. In both instances they will wait a minimum of 100 milliseconds between requests.
+
+In order to change the behavior for API requests, pass a `RetryParams` property to `ClientConfiguration` constructor to a `RetryParams` instance like below with a `MaxRetry` property to control the amount of retries and a `MinWaitInMs` to control the minimum wait time between retried requests.
+
+```csharp
+using OpenFga.Sdk.Client;
+using OpenFga.Sdk.Client.Model;
+using OpenFga.Sdk.Model;
+
+namespace Example {
+    public class Example {
+        public static async Task Main() {
+            try {
+                var configuration = new ClientConfiguration() {
+                    ApiUrl = Environment.GetEnvironmentVariable("FGA_API_URL") ?? "http://localhost:8080", // required, e.g. https://api.fga.example
+                    StoreId = Environment.GetEnvironmentVariable("FGA_STORE_ID"), // not needed when calling `CreateStore` or `ListStores`
+                    AuthorizationModelId = Environment.GetEnvironmentVariable("FGA_AUTHORIZATION_MODEL_ID"), // Optional, can be overridden per request
+                    RetryParams = new RetryParams() {
+                        MaxRetry = 3, // retry up to 3 times on API requests
+                        MinWaitInMs = 250 // wait a minimum of 250 milliseconds between requests
+                    }
+                };
+                var fgaClient = new OpenFgaClient(configuration);
+                var response = await fgaClient.ReadAuthorizationModels();
+            } catch (ApiException e) {
+                 Debug.Print("Error: "+ e);
+            }
+        }
+    }
+}
+```
+
 ### API Endpoints
 
 | Method | HTTP request | Description |

src/OpenFga.Sdk.Test/Api/OpenFgaApiTests.cs

@@ -478,6 +478,102 @@ public class OpenFgaApiTests : IDisposable {
             );
         }
 
+        /// <summary>
+        /// Test that a network calls to get credentials are retried
+        /// </summary>
+        [Fact]
+        public async Task ExchangeCredentialsRetriesTest() {
+            var config = new Configuration.Configuration() {
+                ApiHost = _host,
+                Credentials = new Credentials() {
+                    Method = CredentialsMethod.ClientCredentials,
+                    Config = new CredentialsConfig() {
+                        ClientId = "some-id",
+                        ClientSecret = "some-secret",
+                        ApiTokenIssuer = "tokenissuer.fga.example",
+                        ApiAudience = "some-audience",
+                    }
+                }
+            };
+
+            var mockHandler = new Mock<HttpMessageHandler>(MockBehavior.Strict);
+            mockHandler.Protected()
+                .SetupSequence<Task<HttpResponseMessage>>(
+                    "SendAsync",
+                    ItExpr.Is<HttpRequestMessage>(req =>
+                        req.RequestUri == new Uri($"https://{config.Credentials.Config.ApiTokenIssuer}/oauth/token") &&
+                        req.Method == HttpMethod.Post &&
+                        req.Content.Headers.ContentType.ToString().Equals("application/x-www-form-urlencoded")),
+                    ItExpr.IsAny<CancellationToken>()
+                )
+                .ReturnsAsync(new HttpResponseMessage() {
+                    StatusCode = HttpStatusCode.TooManyRequests
+                })
+                    .ReturnsAsync(new HttpResponseMessage() {
+                        StatusCode = HttpStatusCode.InternalServerError
+                    })
+                .ReturnsAsync(new HttpResponseMessage() {
+                    StatusCode = HttpStatusCode.OK,
+                    Content = Utils.CreateJsonStringContent(new OAuth2Client.AccessTokenResponse() {
+                        AccessToken = "some-token",
+                        ExpiresIn = 86400,
+                        TokenType = "Bearer"
+                    }),
+                });
+
+            var readAuthorizationModelsMockExpression = ItExpr.Is<HttpRequestMessage>(req =>
+                req.RequestUri.ToString()
+                    .StartsWith($"{config.BasePath}/stores/{_storeId}/authorization-models") &&
+                req.Method == HttpMethod.Get &&
+                req.Headers.Contains("Authorization") &&
+                req.Headers.Authorization.Equals(new AuthenticationHeaderValue("Bearer", "some-token")));
+            mockHandler.Protected()
+                .SetupSequence<Task<HttpResponseMessage>>(
+                    "SendAsync",
+                    readAuthorizationModelsMockExpression,
+                    ItExpr.IsAny<CancellationToken>()
+                )
+                .ReturnsAsync(new HttpResponseMessage() {
+                    StatusCode = HttpStatusCode.OK,
+                    Content = Utils.CreateJsonStringContent(
+                            new ReadAuthorizationModelsResponse() { AuthorizationModels = { } }),
+                })
+                .ReturnsAsync(new HttpResponseMessage() {
+                    StatusCode = HttpStatusCode.OK,
+                    Content = Utils.CreateJsonStringContent(
+                            new ReadAuthorizationModelsResponse() { AuthorizationModels = { } }),
+                });
+
+            var httpClient = new HttpClient(mockHandler.Object);
+            var openFgaApi = new OpenFgaApi(config, httpClient);
+
+            var response = await openFgaApi.ReadAuthorizationModels(_storeId, null, null);
+
+            mockHandler.Protected().Verify(
+                "SendAsync",
+                Times.Exactly(3),
+                ItExpr.Is<HttpRequestMessage>(req =>
+                    req.RequestUri == new Uri($"https://{config.Credentials.Config.ApiTokenIssuer}/oauth/token") &&
+                    req.Method == HttpMethod.Post &&
+                    req.Content.Headers.ContentType.ToString().Equals("application/x-www-form-urlencoded")),
+                ItExpr.IsAny<CancellationToken>()
+            );
+            mockHandler.Protected().Verify(
+                "SendAsync",
+                Times.Exactly(1),
+                readAuthorizationModelsMockExpression,
+                ItExpr.IsAny<CancellationToken>()
+            );
+            mockHandler.Protected().Verify(
+                "SendAsync",
+                Times.Exactly(0),
+                ItExpr.Is<HttpRequestMessage>(req =>
+                    req.RequestUri == new Uri($"{config.BasePath}/stores/{_storeId}/check") &&
+                    req.Method == HttpMethod.Post),
+                ItExpr.IsAny<CancellationToken>()
+            );
+        }
+
         /**
          * Errors
          */

src/OpenFga.Sdk/ApiClient/ApiClient.cs

@@ -11,6 +11,7 @@
 //
 
 
+using OpenFga.Sdk.Client.Model;
 using OpenFga.Sdk.Configuration;
 using OpenFga.Sdk.Exceptions;
 
@@ -44,7 +45,7 @@ public class ApiClient : IDisposable {
                 _baseClient = new BaseClient(_configuration, userHttpClient);
                 break;
             case CredentialsMethod.ClientCredentials:
-                _oauth2Client = new OAuth2Client(_configuration.Credentials, _baseClient);
+                _oauth2Client = new OAuth2Client(_configuration.Credentials, _baseClient, new RetryParams { MaxRetry = _configuration.MaxRetry, MinWaitInMs = _configuration.MinWaitInMs });
                 break;
             case CredentialsMethod.None:
             default:

src/OpenFga.Sdk/ApiClient/OAuth2Client.cs

@@ -11,6 +11,7 @@
 //
 
 
+using OpenFga.Sdk.Client.Model;
 using OpenFga.Sdk.Configuration;
 using OpenFga.Sdk.Exceptions;
 using System.Text.Json.Serialization;
@@ -73,6 +74,7 @@ private class AuthToken {
     private AuthToken _authToken = new();
     private IDictionary<string, string> _authRequest { get; set; }
     private string _apiTokenIssuer { get; set; }
+    private RetryParams _retryParams;
 
     #endregion
 
@@ -84,7 +86,7 @@ private class AuthToken {
     /// <param name="credentialsConfig"></param>
     /// <param name="httpClient"></param>
     /// <exception cref="NullReferenceException"></exception>
-    public OAuth2Client(Credentials credentialsConfig, BaseClient httpClient) {
+    public OAuth2Client(Credentials credentialsConfig, BaseClient httpClient, RetryParams retryParams) {
         if (string.IsNullOrWhiteSpace(credentialsConfig.Config!.ClientId)) {
             throw new FgaRequiredParamError("OAuth2Client", "config.ClientId");
         }
@@ -101,6 +103,8 @@ private class AuthToken {
             { "audience", credentialsConfig.Config.ApiAudience },
             { "grant_type", "client_credentials" }
         };
+
+        this._retryParams = retryParams;
     }
 
     /// <summary>
@@ -116,18 +120,47 @@ private class AuthToken {
             Body = Utils.CreateFormEncodedConent(this._authRequest),
         };
 
-        var accessTokenResponse = await _httpClient.SendRequestAsync<AccessTokenResponse>(
+        var accessTokenResponse = await Retry(async () => await _httpClient.SendRequestAsync<AccessTokenResponse>(
             requestBuilder,
             null,
             "ExchangeTokenAsync",
-            cancellationToken);
+            cancellationToken));
 
         _authToken = new AuthToken() {
             AccessToken = accessTokenResponse.AccessToken,
             ExpiresAt = DateTime.Now + TimeSpan.FromSeconds(accessTokenResponse.ExpiresIn)
         };
     }
 
+    private async Task<TResult> Retry<TResult>(Func<Task<TResult>> retryable) {
+        var numRetries = 0;
+        while (true) {
+            try {
+                numRetries++;
+
+                return await retryable();
+            }
+            catch (FgaApiRateLimitExceededError err) {
+                if (numRetries > _retryParams.MaxRetry) {
+                    throw;
+                }
+                var waitInMs = (int)((err.ResetInMs == null || err.ResetInMs < _retryParams.MinWaitInMs)
+                    ? _retryParams.MinWaitInMs
+                    : err.ResetInMs);
+
+                await Task.Delay(waitInMs);
+            }
+            catch (FgaApiError err) {
+                if (!err.ShouldRetry || numRetries > _retryParams.MaxRetry) {
+                    throw;
+                }
+                var waitInMs = _retryParams.MinWaitInMs;
+
+                await Task.Delay(waitInMs);
+            }
+        }
+    }
+
     /// <summary>
     /// Gets the access token, and handles exchanging, rudimentary in memory caching and refreshing it when expired
     /// </summary>