From 044a81c2a78e3decc180bfdb376e5e44533c7fa2 Mon Sep 17 00:00:00 2001 From: Jaap van Blaaderen Date: Mon, 8 Jun 2020 13:00:08 +0200 Subject: [PATCH] fix(lambda): Make retry options configurable for CloudWatchLogs group management fixes #8257 --- packages/@aws-cdk/aws-lambda/lib/function.ts | 13 ++++++- .../lib/log-retention-provider/index.ts | 38 +++++++++++++------ .../@aws-cdk/aws-lambda/lib/log-retention.ts | 26 +++++++++++++ .../test/integ.log-retention.expected.json | 18 ++++----- .../test/test.log-retention-provider.ts | 37 ++++++++++++++++++ 5 files changed, 111 insertions(+), 21 deletions(-) diff --git a/packages/@aws-cdk/aws-lambda/lib/function.ts b/packages/@aws-cdk/aws-lambda/lib/function.ts index ea2d2bf1f18ef..c5539fd3c4f37 100644 --- a/packages/@aws-cdk/aws-lambda/lib/function.ts +++ b/packages/@aws-cdk/aws-lambda/lib/function.ts @@ -12,7 +12,7 @@ import { calculateFunctionHash, trimFromStart } from './function-hash'; import { Version, VersionOptions } from './lambda-version'; import { CfnFunction } from './lambda.generated'; import { ILayerVersion } from './layers'; -import { LogRetention } from './log-retention'; +import { LogRetention, LogRetentionRetryOptions } from './log-retention'; import { Runtime } from './runtime'; /** @@ -232,6 +232,16 @@ export interface FunctionOptions extends EventInvokeConfigOptions { */ readonly logRetentionRole?: iam.IRole; + /** + * Retry options for creating CloudWatch log groups. Deploying many Lambdas + * with log retention resources may result in rate limit issues when creating + * CloudWatch Log groups. The retry options allow you to customize the retry + * options in order to successfully create these. + * + * @default - Default retry options of the AWS SDK. + */ + readonly logRetentionRetryOptions?: LogRetentionRetryOptions; + /** * Options for the `lambda.Version` resource automatically created by the * `fn.currentVersion` method. @@ -544,6 +554,7 @@ export class Function extends FunctionBase { logGroupName: `/aws/lambda/${this.functionName}`, retention: props.logRetention, role: props.logRetentionRole, + logRetentionRetryOptions: props.logRetentionRetryOptions, }); this._logGroup = logs.LogGroup.fromLogGroupArn(this, 'LogGroup', logretention.logGroupArn); } diff --git a/packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts b/packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts index 5e4b3e38a34d8..668913e76d9fd 100644 --- a/packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts +++ b/packages/@aws-cdk/aws-lambda/lib/log-retention-provider/index.ts @@ -2,18 +2,16 @@ // eslint-disable-next-line import/no-extraneous-dependencies import * as AWS from 'aws-sdk'; - -// Ensure we do not run into throttling issues when deploying stack(s) with a lot of Lambdas. -const retryOptions = { maxRetries: 6, retryDelayOptions: { base: 300 }}; +import { LogRetentionRetryOptions } from '../log-retention'; /** * Creates a log group and doesn't throw if it exists. * * @param logGroupName the name of the log group to create */ -async function createLogGroupSafe(logGroupName: string) { +async function createLogGroupSafe(logGroupName: string, retryOptions?: LogRetentionRetryOptions) { try { // Try to create the log group - const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions}); + const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions }); await cloudwatchlogs.createLogGroup({ logGroupName }).promise(); } catch (e) { if (e.code !== 'ResourceAlreadyExistsException') { @@ -28,8 +26,8 @@ async function createLogGroupSafe(logGroupName: string) { * @param logGroupName the name of the log group to create * @param retentionInDays the number of days to retain the log events in the specified log group. */ -async function setRetentionPolicy(logGroupName: string, retentionInDays?: number) { - const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions}); +async function setRetentionPolicy(logGroupName: string, retryOptions?: LogRetentionRetryOptions, retentionInDays?: number) { + const cloudwatchlogs = new AWS.CloudWatchLogs({ apiVersion: '2014-03-28', ...retryOptions }); if (!retentionInDays) { await cloudwatchlogs.deleteRetentionPolicy({ logGroupName }).promise(); } else { @@ -44,10 +42,13 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent // The target log group const logGroupName = event.ResourceProperties.LogGroupName; + // Parse retry options for creating the target log group + const retryOptions = parseRetryOptions(event.ResourceProperties.LogRetentionRetryOptions); + if (event.RequestType === 'Create' || event.RequestType === 'Update') { // Act on the target log group - await createLogGroupSafe(logGroupName); - await setRetentionPolicy(logGroupName, parseInt(event.ResourceProperties.RetentionInDays, 10)); + await createLogGroupSafe(logGroupName, retryOptions); + await setRetentionPolicy(logGroupName, retryOptions, parseInt(event.ResourceProperties.RetentionInDays, 10)); if (event.RequestType === 'Create') { // Set a retention policy of 1 day on the logs of this function. The log @@ -59,8 +60,8 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent // same time. This can sometime result in an OperationAbortedException. To // avoid this and because this operation is not critical we catch all errors. try { - await createLogGroupSafe(`/aws/lambda/${context.functionName}`); - await setRetentionPolicy(`/aws/lambda/${context.functionName}`, 1); + await createLogGroupSafe(`/aws/lambda/${context.functionName}`, retryOptions); + await setRetentionPolicy(`/aws/lambda/${context.functionName}`, retryOptions, 1); } catch (e) { console.log(e); } @@ -111,4 +112,19 @@ export async function handler(event: AWSLambda.CloudFormationCustomResourceEvent } }); } + + function parseRetryOptions(rawOptions: any) { + const retryOptions: { maxRetries?: number, retryOptions?: { base?: number } } = {}; + if (rawOptions) { + if (rawOptions.maxRetries) { + retryOptions.maxRetries = parseInt(rawOptions.maxRetries, 10); + } + if (rawOptions.base) { + retryOptions.retryOptions = { + base: parseInt(rawOptions.base, 10), + }; + } + } + return retryOptions; + } } diff --git a/packages/@aws-cdk/aws-lambda/lib/log-retention.ts b/packages/@aws-cdk/aws-lambda/lib/log-retention.ts index 74feeb5e62794..9918bf8aa3df6 100644 --- a/packages/@aws-cdk/aws-lambda/lib/log-retention.ts +++ b/packages/@aws-cdk/aws-lambda/lib/log-retention.ts @@ -26,6 +26,31 @@ export interface LogRetentionProps { * @default - A new role is created */ readonly role?: iam.IRole; + + /** + * Retry options for managing CloudWatch log groups. + * + * @default - AWS SDK default retry options + */ + readonly logRetentionRetryOptions?: LogRetentionRetryOptions; +} + +/** + * Retry options for managing CloudWatch log groups + */ +export interface LogRetentionRetryOptions { + /** + * The maximum amount of retries. + * + * @default - AWS SDK default + */ + readonly maxRetries?: number; + /** + * The base number of milliseconds to use in the exponential backoff for operation retries. + * + * @default - AWS SDK default + */ + readonly base?: number; } /** @@ -69,6 +94,7 @@ export class LogRetention extends cdk.Construct { properties: { ServiceToken: provider.functionArn, LogGroupName: props.logGroupName, + LogRetentionRetryOptions: props.logRetentionRetryOptions, RetentionInDays: props.retention === logs.RetentionDays.INFINITE ? undefined : props.retention, }, }); diff --git a/packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json b/packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json index ff3bb7285e69f..d33ff3f57d2eb 100644 --- a/packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json +++ b/packages/@aws-cdk/aws-lambda/test/integ.log-retention.expected.json @@ -133,7 +133,7 @@ "Properties": { "Code": { "S3Bucket": { - "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3BucketD782C750" + "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3BucketA7A09DD7" }, "S3Key": { "Fn::Join": [ @@ -146,7 +146,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196" + "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3" } ] } @@ -159,7 +159,7 @@ "Fn::Split": [ "||", { - "Ref": "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196" + "Ref": "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3" } ] } @@ -331,17 +331,17 @@ } }, "Parameters": { - "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3BucketD782C750": { + "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3BucketA7A09DD7": { "Type": "String", - "Description": "S3 bucket for asset \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\"" + "Description": "S3 bucket for asset \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\"" }, - "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beS3VersionKeyB87E9196": { + "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fS3VersionKey579A73D3": { "Type": "String", - "Description": "S3 key for asset version \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\"" + "Description": "S3 key for asset version \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\"" }, - "AssetParametersc11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6beArtifactHashBA1C5764": { + "AssetParameterse375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028fArtifactHashE75963CF": { "Type": "String", - "Description": "Artifact hash for asset \"c11219c29950dc50a505625251bd4e6b553c3d85f04bcba46572f2e25e8fe6be\"" + "Description": "Artifact hash for asset \"e375da66e6ab168bb13b858a043e9e8c8c20334b443d746983fa4ad0dcc7028f\"" } } } \ No newline at end of file diff --git a/packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts b/packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts index bfc37c0d1b6f2..02c8cbfa08c3a 100644 --- a/packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts +++ b/packages/@aws-cdk/aws-lambda/test/test.log-retention-provider.ts @@ -296,4 +296,41 @@ export = { test.done(); }, + + async 'custom log retention retry options'(test: Test) { + AWS.mock('CloudWatchLogs', 'createLogGroup', sinon.fake.resolves({})); + AWS.mock('CloudWatchLogs', 'putRetentionPolicy', sinon.fake.resolves({})); + AWS.mock('CloudWatchLogs', 'deleteRetentionPolicy', sinon.fake.resolves({})); + + const event = { + ...eventCommon, + RequestType: 'Create', + ResourceProperties: { + ServiceToken: 'token', + RetentionInDays: '30', + LogGroupName: 'group', + LogRetentionRetryOptions: { + maxRetries: '5', + base: '300', + }, + }, + }; + + const request = createRequest('SUCCESS'); + + await provider.handler(event as AWSLambda.CloudFormationCustomResourceCreateEvent, context); + + sinon.assert.calledWith(AWSSDK.CloudWatchLogs as any, { + apiVersion: '2014-03-28', + maxRetries: 5, + retryOptions: { + base: 300, + }, + }); + + test.equal(request.isDone(), true); + + test.done(); + }, + };