feat: Retry fetch requests and disable dispatch on failure. (#174)

src/dispatch/Dispatch.ts

@@ -6,6 +6,7 @@ import { FetchHttpHandler } from './FetchHttpHandler';
 import { PutRumEventsRequest } from './dataplane';
 import { Config } from '../orchestration/Orchestration';
 import { v4 } from 'uuid';
+import { RetryHttpHandler } from './RetryHttpHandler';
 
 type SendFunction = (
     putRumEventsRequest: PutRumEventsRequest
@@ -101,7 +102,7 @@ export class Dispatch {
      * Send meta data and events to the AWS RUM data plane service via fetch.
      */
     public dispatchFetch = async (): Promise<{ response: HttpResponse }> => {
-        return this.dispatch(this.rum.sendFetch);
+        return this.dispatch(this.rum.sendFetch).catch(this.handleReject);
     };
 
     /**
@@ -189,6 +190,14 @@ export class Dispatch {
         return send(putRumEventsRequest);
     }
 
+    private handleReject = (e: any): { response: HttpResponse } => {
+        // The handler has run out of retries. We adhere to our convention to
+        // fail safe by disabling dispatch. This ensures that we will not
+        // continue to attempt requests when the problem is not recoverable.
+        this.disable();
+        throw e;
+    };
+
     /**
      * The default method for creating data plane service clients.
      * @param endpoint Service endpoint.
@@ -201,9 +210,12 @@ export class Dispatch {
         credentials
     ) => {
         return new DataPlaneClient({
-            fetchRequestHandler: new FetchHttpHandler({
-                fetchFunction: this.config.fetchFunction
-            }),
+            fetchRequestHandler: new RetryHttpHandler(
+                new FetchHttpHandler({
+                    fetchFunction: this.config.fetchFunction
+                }),
+                this.config.retries
+            ),
             beaconRequestHandler: new BeaconHttpHandler(),
             endpoint,
             region,

src/dispatch/RetryHttpHandler.ts

@@ -0,0 +1,56 @@
+import { HttpHandler, HttpRequest, HttpResponse } from '@aws-sdk/protocol-http';
+
+export type BackoffFunction = (retry: number) => number;
+
+/**
+ * An HttpHandler which wraps other HttpHandlers to retry requests.
+ *
+ * Requests will be retried if (1) there is an error (e.g., with the network or
+ * credentials) and the promise rejects, or (2) the response status is not 2xx.
+ */
+export class RetryHttpHandler implements HttpHandler {
+    private handler: HttpHandler;
+    private retries: number;
+    private backoff: BackoffFunction;
+
+    public constructor(
+        handler: HttpHandler,
+        retries: number,
+        backoff: BackoffFunction = (n) => n * 2000
+    ) {
+        this.handler = handler;
+        this.retries = retries;
+        this.backoff = backoff;
+    }
+
+    public async handle(
+        request: HttpRequest
+    ): Promise<{ response: HttpResponse }> {
+        let retriesLeft = this.retries;
+        while (true) {
+            try {
+                const response = await this.handler.handle(request);
+                if (this.isStatusCode2xx(response.response.statusCode)) {
+                    return response;
+                }
+                throw new Error(`${response.response.statusCode}`);
+            } catch (e) {
+                if (!retriesLeft) {
+                    throw e;
+                }
+                retriesLeft--;
+                await this.sleep(this.backoff(this.retries - retriesLeft));
+            }
+        }
+    }
+
+    private async sleep(milliseconds): Promise<void> {
+        return new Promise<void>((resolve) =>
+            setTimeout(resolve, milliseconds)
+        );
+    }
+
+    private isStatusCode2xx = (statusCode: number): boolean => {
+        return statusCode >= 200 && statusCode < 300;
+    };
+}

src/dispatch/__tests__/Dispatch.test.ts

@@ -3,6 +3,7 @@ import * as Utils from '../../test-utils/test-utils';
 import { DataPlaneClient } from '../DataPlaneClient';
 import { CredentialProvider } from '@aws-sdk/types';
 import { DEFAULT_CONFIG, mockFetch } from '../../test-utils/test-utils';
+import { EventCache } from 'event-cache/EventCache';
 
 global.fetch = mockFetch;
 const sendFetch = jest.fn(() => Promise.resolve());
@@ -67,7 +68,7 @@ describe('Dispatch tests', () => {
         expect(credentialProvider).toHaveBeenCalledTimes(1);
     });
 
-    test('dispatch() throws exception when LogEventsCommand fails', async () => {
+    test('dispatch() throws exception when send fails', async () => {
         // Init
         const sendFetch = jest.fn(() =>
             Promise.reject('Something went wrong.')
@@ -373,4 +374,35 @@ describe('Dispatch tests', () => {
             undefined
         );
     });
+
+    test('when a fetch request is rejected then dispatch is disabled', async () => {
+        // Init
+        const ERROR = 'Something went wrong.';
+        const sendFetch = jest.fn(() => Promise.reject(ERROR));
+        (DataPlaneClient as any).mockImplementation(() => {
+            return {
+                sendFetch
+            };
+        });
+
+        const eventCache: EventCache = Utils.createDefaultEventCacheWithEvents();
+
+        const dispatch = new Dispatch(
+            Utils.AWS_RUM_REGION,
+            Utils.AWS_RUM_ENDPOINT,
+            eventCache,
+            {
+                ...DEFAULT_CONFIG,
+                ...{ dispatchInterval: Utils.AUTO_DISPATCH_OFF, retries: 0 }
+            }
+        );
+        dispatch.setAwsCredentials(Utils.createAwsCredentials());
+
+        // Run
+        await expect(dispatch.dispatchFetch()).rejects.toEqual(ERROR);
+        eventCache.recordEvent('com.amazon.rum.event1', {});
+
+        // Assert
+        await expect(dispatch.dispatchFetch()).resolves.toEqual(undefined);
+    });
 });