fix: retry failed google-auth-library requests

Adds a custom `gaxios` config to be used in `google-auth-library`
requests, setting a proper retry value. This should fix the flaky tests
on CI and provide a more resilient solution to end users.

Fix: #134
Fix: #146

package.json

@@ -86,6 +86,7 @@
   },
   "dependencies": {
     "@googleapis/sqladmin": "^10.0.0",
+    "gaxios": "^5.1.3",
     "google-auth-library": "^8.9.0"
   }
 }

src/sqladmin-fetcher.ts

@@ -14,6 +14,7 @@
 
 import {GoogleAuth} from 'google-auth-library';
 import {sqladmin_v1beta4} from '@googleapis/sqladmin';
+import {instance as gaxios} from 'gaxios';
 const {Sqladmin} = sqladmin_v1beta4;
 import {InstanceConnectionInfo} from './instance-connection-info';
 import {SslCert} from './ssl-cert';
@@ -33,6 +34,35 @@ interface RequestBody {
   access_token?: string;
 }
 
+// Default values for the list of http methods to retry in gaxios
+// ref: https://github.com/googleapis/gaxios/tree/fdb6a8e542f7782f8d1e487c0ef7d301fa231d18#request-options
+const defaultGaxiosHttpMethodsToRetry =
+  ['GET', 'HEAD', 'PUT', 'OPTIONS', 'DELETE'];
+
+// https://github.com/googleapis/gaxios is the http request library used by
+// google-auth-library and other Cloud SDK libraries, this function will set
+// a standard default configuration that will ensure retry works as expected
+// for internal google-auth-library requests.
+function setupGaxiosConfig() {
+  gaxios.defaults = {
+    retryConfig: {
+      retry: 3,
+      // Make sure to add POST to the list of default methods to retry
+      // since it's used in IAM generateAccessToken requests that needs retry
+      httpMethodsToRetry: ['POST', ...defaultGaxiosHttpMethodsToRetry],
+      // Should retry on non-http error codes such as ECONNRESET, ETIMEOUT, etc
+      noResponseRetries: 3,
+    },
+  };
+}
+
+const defaultGaxiosConfig = gaxios.defaults;
+// resumes the previous default gaxios config in order to reduce the chance of
+// affecting other libraries that might be sharing that same gaxios instance
+function cleanGaxiosConfig() {
+  gaxios.defaults = defaultGaxiosConfig;
+}
+
 export class SQLAdminFetcher {
   private readonly client: sqladmin_v1beta4.Sqladmin;
   private readonly auth: GoogleAuth;
@@ -63,6 +93,8 @@ export class SQLAdminFetcher {
     regionId,
     instanceId,
   }: InstanceConnectionInfo): Promise<InstanceMetadata> {
+    setupGaxiosConfig();
+
     const res = await this.client.connect.get({
       project: projectId,
       instance: instanceId,
@@ -105,6 +137,8 @@ export class SQLAdminFetcher {
       });
     }
 
+    cleanGaxiosConfig();
+
     return {
       ipAddresses,
       serverCaCert: {
@@ -119,6 +153,8 @@ export class SQLAdminFetcher {
     publicKey: string,
     authType: AuthTypes
   ): Promise<SslCert> {
+    setupGaxiosConfig();
+
     const requestBody: RequestBody = {
       public_key: publicKey,
     };
@@ -172,6 +208,8 @@ export class SQLAdminFetcher {
       tokenExpiration
     );
 
+    cleanGaxiosConfig();
+
     return {
       cert,
       expirationTime: nearestExpiration,

test/sqladmin-fetcher.ts

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import {resolve} from 'node:path';
 import t from 'tap';
 import nock from 'nock';
 import {GoogleAuth} from 'google-auth-library';
@@ -381,3 +382,65 @@ t.test('getEphemeralCertificate sets access token', async t => {
     'should return earlier token expiration time'
   );
 });
+
+t.test('generateAccessToken endpoint should retry', async t => {
+  const path = t.testdir({
+    credentials: JSON.stringify({
+      delegates: [],
+      service_account_impersonation_url:
+        'https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/foo@dev.org:generateAccessToken',
+      source_credentials: {
+        client_id:
+          'b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c',
+        client_secret:
+          '7d865e959b2466918c9863afca942d0fb89d7c9ac0c99bafc3749504ded97730',
+        refresh_token:
+          'bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c',
+        type: 'authorized_user',
+      },
+      type: 'impersonated_service_account',
+    }),
+  });
+  const creds = process.env.GOOGLE_APPLICATION_CREDENTIALS;
+  process.env.GOOGLE_APPLICATION_CREDENTIALS = resolve(path, 'credentials');
+  t.teardown(() => {
+    process.env.GOOGLE_APPLICATION_CREDENTIALS = creds;
+  });
+
+  nock('https://oauth2.googleapis.com')
+    .persist()
+    .post('/token')
+    .reply(200, {access_token: 'abc123', expires_in: 1});
+
+  nock('https://oauth2.googleapis.com').post('/tokeninfo').reply(200, {
+    expires_in: 3600,
+    scope: 'https://www.googleapis.com/auth/sqlservice.login',
+  });
+
+  nock('https://iamcredentials.googleapis.com/v1')
+    .post('/projects/-/serviceAccounts/foo@dev.org:generateAccessToken')
+    .replyWithError({code: 'ECONNRESET'});
+
+  nock('https://iamcredentials.googleapis.com/v1')
+    .post('/projects/-/serviceAccounts/foo@dev.org:generateAccessToken')
+    .reply(200, {
+      accessToken:
+        'b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c',
+      expireTime: new Date(Date.now() + 3600).toISOString(),
+    });
+
+  const instanceConnectionInfo: InstanceConnectionInfo = {
+    projectId: 'my-project',
+    regionId: 'us-east1',
+    instanceId: 'my-instance',
+  };
+
+  // Second try should succeed
+  mockRequest(instanceConnectionInfo);
+
+  const fetcher = new SQLAdminFetcher();
+  const instanceMetadata = await fetcher.getInstanceMetadata(
+    instanceConnectionInfo
+  );
+  t.ok(instanceMetadata, 'should return expected instance metadata object');
+});