fix: handle RequestLimitExceeded error

hoonoh · Dec 7, 2020 · ed475c9 · ed475c9
1 parent db3b23b
commit ed475c9
Show file tree

Hide file tree

Showing 4 changed files with 414 additions and 20 deletions.
diff --git a/src/lib/core.spec.ts b/src/lib/core.spec.ts
@@ -1,6 +1,7 @@
 import mockConsole, { RestoreConsole } from 'jest-mock-console';
 import { filter } from 'lodash';
 import nock from 'nock';
+import { stdout } from 'process';
 
 import { mockAwsCredentials, mockAwsCredentialsClear } from '../../test/mock-credential-endpoints';
 import {
@@ -260,6 +261,27 @@ describe('lib', () => {
       });
     });
 
+    describe('should handle RequestLimitExceeded error', () => {
+      const region = 'us-east-1';
+      let results: SpotPriceExtended[];
+      let restoreConsole: RestoreConsole;
+
+      beforeAll(async () => {
+        restoreConsole = mockConsole();
+        mockDefaultRegionEndpoints({ returnRequestLimitExceededErrorCount: 10 });
+        results = await getGlobalSpotPrices({ regions: [region] });
+      });
+
+      afterAll(() => {
+        restoreConsole();
+        mockDefaultRegionEndpointsClear();
+      });
+
+      it('should return expected values', async () => {
+        expect(results).toMatchSnapshot();
+      });
+    });
+
     describe('should fetch ec2 instance type info dynamically if not found from constants', () => {
       let results: SpotPriceExtended[];
       let restoreConsole: RestoreConsole;
@@ -346,5 +368,27 @@ describe('lib', () => {
         expect(results).toMatchSnapshot();
       });
     });
+
+    describe('should handle RequestLimitExceeded error', () => {
+      let results: GetEc2InfoResults;
+
+      beforeAll(async () => {
+        mockDefaultRegionEndpoints({
+          returnRequestLimitExceededErrorCount: 10,
+          maxLength: 5,
+          returnPartialBlankValues: true,
+        });
+        results = await getEc2Info({ InstanceTypes: ['dummy.large'] });
+      });
+
+      afterAll(() => {
+        mockDefaultRegionEndpointsClear();
+      });
+
+      it('should return expected values', () => {
+        expect(Object.keys(results).length).toEqual(1);
+        expect(results).toMatchSnapshot();
+      });
+    });
   });
 });
diff --git a/src/lib/core.ts b/src/lib/core.ts
@@ -1,4 +1,6 @@
 import EC2 from 'aws-sdk/clients/ec2';
+import { AWSError } from 'aws-sdk/lib/error';
+import { PromiseResult } from 'aws-sdk/lib/request';
 import { ec2Info, Ec2InstanceInfo } from '../constants/ec2-info';
 
 import { InstanceFamilyType, InstanceSize, InstanceType } from '../constants/ec2-types';
@@ -82,18 +84,36 @@ const getEc2SpotPrice = async (options: {
       secretAccessKey,
     });
 
+    const startTime = new Date();
+    startTime.setHours(startTime.getHours() - 3);
+
     const fetch = async (nextToken?: string): Promise<EC2.SpotPrice[]> => {
-      const startTime = new Date();
-      startTime.setHours(startTime.getHours() - 3);
-
-      const result = await ec2
-        .describeSpotPriceHistory({
-          NextToken: nextToken,
-          StartTime: startTime,
-          ProductDescriptions: platforms,
-          InstanceTypes: instanceTypes,
-        })
-        .promise();
+      let retryMS = 0;
+
+      const describeSpotPriceHistory = async (): Promise<
+        PromiseResult<EC2.DescribeSpotPriceHistoryResult, AWSError>
+      > => {
+        try {
+          return await ec2
+            .describeSpotPriceHistory({
+              NextToken: nextToken,
+              StartTime: startTime,
+              ProductDescriptions: platforms,
+              InstanceTypes: instanceTypes,
+            })
+            .promise();
+        } catch (error) {
+          if (error.code === 'RequestLimitExceeded') {
+            retryMS = retryMS ? retryMS * 2 : 200;
+            await new Promise(res => setTimeout(res, retryMS));
+            return describeSpotPriceHistory();
+          } else {
+            throw error;
+          }
+        }
+      };
+
+      const result = await describeSpotPriceHistory();
 
       const nextList = result.NextToken ? await fetch(result.NextToken) : [];
 
@@ -144,14 +164,34 @@ export const getEc2Info = async ({
   const ec2 = new EC2({ region });
 
   const fetchInfo = async (NextToken?: string): Promise<Ec2InstanceInfos> => {
+    let retryMS = 0;
+
     const rtn: Ec2InstanceInfos = {};
-    const res = await ec2
-      .describeInstanceTypes({
-        NextToken,
-        MaxResults: InstanceTypes ? undefined : 100,
-        InstanceTypes,
-      })
-      .promise();
+
+    const describeInstanceTypes = async (): Promise<
+      PromiseResult<EC2.DescribeInstanceTypesResult, AWSError>
+    > => {
+      try {
+        return await ec2
+          .describeInstanceTypes({
+            NextToken,
+            MaxResults: InstanceTypes ? undefined : 100,
+            InstanceTypes,
+          })
+          .promise();
+      } catch (error) {
+        if (error.code === 'RequestLimitExceeded') {
+          retryMS = retryMS ? retryMS * 2 : 200;
+          await new Promise(res => setTimeout(res, retryMS));
+          return describeInstanceTypes();
+        } else {
+          throw error;
+        }
+      }
+    };
+
+    const res = await describeInstanceTypes();
+
     res.InstanceTypes?.forEach(i => {
       if (i.InstanceType) {
         rtn[i.InstanceType] = {