fix(lambda): Make retry options configurable for CloudWatchLogs group…

… management

fixes #8257

packages/@aws-cdk/aws-lambda/lib/function.ts

@@ -12,7 +12,7 @@ import { calculateFunctionHash, trimFromStart } from './function-hash';
 import { Version, VersionOptions } from './lambda-version';
 import { CfnFunction } from './lambda.generated';
 import { ILayerVersion } from './layers';
-import { LogRetention } from './log-retention';
+import { LogRetention, LogRetentionRetryOptions } from './log-retention';
 import { Runtime } from './runtime';
 
 /**
@@ -232,6 +232,16 @@ export interface FunctionOptions extends EventInvokeConfigOptions {
    */
   readonly logRetentionRole?: iam.IRole;
 
+  /**
+   * Retry options for creating CloudWatch log groups. Deploying many Lambdas
+   * with log retention resources may result in rate limit issues when creating
+   * CloudWatch Log groups. The retry options allow you to customize the retry
+   * options in order to successfully create these.
+   *
+   * @default - Default retry options of the AWS SDK.
+   */
+  readonly logRetentionRetryOptions?: LogRetentionRetryOptions;
+
   /**
    * Options for the `lambda.Version` resource automatically created by the
    * `fn.currentVersion` method.
@@ -544,6 +554,7 @@ export class Function extends FunctionBase {
         logGroupName: `/aws/lambda/${this.functionName}`,
         retention: props.logRetention,
         role: props.logRetentionRole,
+        logRetentionRetryOptions: props.logRetentionRetryOptions,
       });
       this._logGroup = logs.LogGroup.fromLogGroupArn(this, 'LogGroup', logretention.logGroupArn);
     }

packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts

@@ -2,18 +2,16 @@
 
 // eslint-disable-next-line import/no-extraneous-dependencies
 import * as AWS from 'aws-sdk';
-
-// Ensure we do not run into throttling issues when deploying stack(s) with a lot of Lambdas.
-const retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }};
+import { LogRetentionRetryOptions } from '../log-retention';
 
 /**
  * Creates a log group and doesn't throw if it exists.
  *
  * @param logGroupName the name of the log group to create
  */
-async function createLogGroupSafe(logGroupName: string) {
+async function createLogGroupSafe(logGroupName: string, retryOptions?: LogRetentionRetryOptions) {
   try { // Try to create the log group
-    const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions});
+    const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions });
     await cloudwatchlogs.createLogGroup({ logGroupName }).promise();
   } catch (e) {
     if (e.code !== 'ResourceAlreadyExistsException') {
@@ -28,8 +26,8 @@ async function createLogGroupSafe(logGroupName: string) {
  * @param logGroupName the name of the log group to create
  * @param retentionInDays the number of days to retain the log events in the specified log group.
  */
-async function setRetentionPolicy(logGroupName: string, retentionInDays?: number) {
-  const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions});
+async function setRetentionPolicy(logGroupName: string, retryOptions?: LogRetentionRetryOptions, retentionInDays?: number) {
+  const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions });
   if (!retentionInDays) {
     await cloudwatchlogs.deleteRetentionPolicy({ logGroupName }).promise();
   } else {
@@ -44,10 +42,13 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
     // The target log group
     const logGroupName = event.ResourceProperties.LogGroupName;
 
+    // Parse retry options for creating the target log group
+    const retryOptions = parseRetryOptions(event.ResourceProperties.LogRetentionRetryOptions);
+
     if (event.RequestType === 'Create' || event.RequestType === 'Update') {
       // Act on the target log group
-      await createLogGroupSafe(logGroupName);
-      await setRetentionPolicy(logGroupName, parseInt(event.ResourceProperties.RetentionInDays, 10));
+      await createLogGroupSafe(logGroupName, retryOptions);
+      await setRetentionPolicy(logGroupName, retryOptions, parseInt(event.ResourceProperties.RetentionInDays, 10));
 
       if (event.RequestType === 'Create') {
         // Set a retention policy of 1 day on the logs of this function. The log
@@ -59,8 +60,8 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
         // same time. This can sometime result in an OperationAbortedException. To
         // avoid this and because this operation is not critical we catch all errors.
         try {
-          await createLogGroupSafe(`/aws/lambda/${context.functionName}`);
-          await setRetentionPolicy(`/aws/lambda/${context.functionName}`, 1);
+          await createLogGroupSafe(`/aws/lambda/${context.functionName}`, retryOptions);
+          await setRetentionPolicy(`/aws/lambda/${context.functionName}`, retryOptions, 1);
         } catch (e) {
           console.log(e);
         }
@@ -111,4 +112,19 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent
       }
     });
   }
+
+  function parseRetryOptions(rawOptions: any) {
+    const retryOptions: { maxRetries?: number, retryOptions?: { base?: number } } = {};
+    if (rawOptions) {
+      if (rawOptions.maxRetries) {
+        retryOptions.maxRetries = parseInt(rawOptions.maxRetries, 10);
+      }
+      if (rawOptions.base) {
+        retryOptions.retryOptions = {
+          base: parseInt(rawOptions.base, 10),
+        };
+      }
+    }
+    return retryOptions;
+  }
 }

packages/@aws-cdk/aws-lambda/lib/log-retention.ts

@@ -26,6 +26,31 @@ export interface LogRetentionProps {
    * @default - A new role is created
    */
   readonly role?: iam.IRole;
+
+  /**
+   * Retry options for managing CloudWatch log groups.
+   *
+   * @default - AWS SDK default retry options
+   */
+  readonly logRetentionRetryOptions?: LogRetentionRetryOptions;
+}
+
+/**
+ * Retry options for managing CloudWatch log groups
+ */
+export interface LogRetentionRetryOptions {
+  /**
+   * The maximum amount of retries.
+   *
+   * @default - AWS SDK default
+   */
+  readonly maxRetries?: number;
+  /**
+   * The base number of milliseconds to use in the exponential backoff for operation retries.
+   *
+   * @default - AWS SDK default
+   */
+  readonly base?: number;
 }
 
 /**
@@ -69,6 +94,7 @@ export class LogRetention extends cdk.Construct {
       properties: {
         ServiceToken: provider.functionArn,
         LogGroupName: props.logGroupName,
+        LogRetentionRetryOptions: props.logRetentionRetryOptions,
         RetentionInDays: props.retention === logs.RetentionDays.INFINITE ? undefined : props.retention,
       },
     });

packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json

@@ -133,7 +133,7 @@
       "Properties": {
         "Code": {
           "S3Bucket": {
-            "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3BucketD782C750"
+            "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3BucketA7A09DD7"
           },
           "S3Key": {
             "Fn::Join": [
@@ -146,7 +146,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196"
+                          "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3"
                         }
                       ]
                     }
@@ -159,7 +159,7 @@
                       "Fn::Split": [
                         "||",
                         {
-                          "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196"
+                          "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3"
                         }
                       ]
                     }
@@ -331,17 +331,17 @@
     }
   },
   "Parameters": {
-    "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3BucketD782C750": {
+    "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3BucketA7A09DD7": {
       "Type": "String",
-      "Description": "S3 bucket for asset \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\""
+      "Description": "S3 bucket for asset \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\""
     },
-    "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196": {
+    "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3": {
       "Type": "String",
-      "Description": "S3 key for asset version \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\""
+      "Description": "S3 key for asset version \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\""
     },
-    "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beArtifactHashBA1C5764": {
+    "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fArtifactHashE75963CF": {
       "Type": "String",
-      "Description": "Artifact hash for asset \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\""
+      "Description": "Artifact hash for asset \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\""
     }
   }
 }

packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts

@@ -296,4 +296,41 @@ export = {
 
     test.done();
   },
+
+  async 'custom log retention retry options'(test: Test) {
+    AWS.mock('CloudWatchLogs', 'createLogGroup', sinon.fake.resolves({}));
+    AWS.mock('CloudWatchLogs', 'putRetentionPolicy', sinon.fake.resolves({}));
+    AWS.mock('CloudWatchLogs', 'deleteRetentionPolicy', sinon.fake.resolves({}));
+
+    const event = {
+      ...eventCommon,
+      RequestType: 'Create',
+      ResourceProperties: {
+        ServiceToken: 'token',
+        RetentionInDays: '30',
+        LogGroupName: 'group',
+        LogRetentionRetryOptions: {
+          maxRetries: '5',
+          base: '300',
+        },
+      },
+    };
+
+    const request = createRequest('SUCCESS');
+
+    await provider.handler(event as AWSLambda.CloudFormationCustomResourceCreateEvent, context);
+
+    sinon.assert.calledWith(AWSSDK.CloudWatchLogs as any, {
+      apiVersion: '2014-03-28',
+      maxRetries: 5,
+      retryOptions: {
+        base: 300,
+      },
+    });
+
+    test.equal(request.isDone(), true);
+
+    test.done();
+  },
+
 };