diff --git a/reference-artifacts/Add-ons/opensiem/lambdas/common/src/backoff.ts b/reference-artifacts/Add-ons/opensiem/lambdas/common/src/backoff.ts index bec056d47..23ee07437 100644 --- a/reference-artifacts/Add-ons/opensiem/lambdas/common/src/backoff.ts +++ b/reference-artifacts/Add-ons/opensiem/lambdas/common/src/backoff.ts @@ -20,8 +20,24 @@ export function throttlingBackOff( request: () => Promise, options?: Partial>, ): Promise { + const defaultDelay = 500; + let maxDelayValue = 2000; + + if (process.env.BACKOFF_START_DELAY) { + const backoffStartDelay = parseInt(process.env.BACKOFF_START_DELAY, 10); + if (Number.isInteger(backoffStartDelay)) { + maxDelayValue = backoffStartDelay; + } + } + + // Add jitter to the starting delay + const startingDelay = Math.random() * (maxDelayValue - defaultDelay + 1) + defaultDelay; + + console.log(`throttlingBackOff delay set to ${startingDelay}`); + return backOff(request, { - startingDelay: 500, + startingDelay, + delayFirstAttempt: true, jitter: 'full', retry: isThrottlingError, ...options, @@ -34,6 +50,7 @@ export const isThrottlingError = (e: any) => e.code === 'ConcurrentModificationException' || // Retry for AWS Organizations e.code === 'InsufficientDeliveryPolicyException' || // Retry for ConfigService e.code === 'NoAvailableDeliveryChannelException' || // Retry for ConfigService + e.code === 'ConcurrentModifications' || // Retry for AssociateHostedZone e.code === 'TooManyRequestsException' || e.code === 'Throttling' || e.code === 'ThrottlingException' || diff --git a/src/core/cdk/src/index.ts b/src/core/cdk/src/index.ts index a755d7670..cd57b9759 100644 --- a/src/core/cdk/src/index.ts +++ b/src/core/cdk/src/index.ts @@ -39,6 +39,7 @@ async function main() { env.CONFIG_S3_BUCKET || `${acceleratorPrefix.toLowerCase()}${cdk.Aws.ACCOUNT_ID}-${cdk.Aws.REGION}-config`; const codebuildComputeType = env.BUILD_COMPUTE_TYPE || 'BUILD_GENERAL1_LARGE'; const pageSize = env.DEPLOY_STACK_PAGE_SIZE || '900'; + const backoff = env.BACKOFF_START_DELAY; const enablePrebuiltProject = 'ENABLE_PREBUILT_PROJECT' in env; const notificationEmail = env.NOTIFICATION_EMAIL || 'notify@example.com'; const installerCmk = env.INSTALLER_CMK || `alias/${acceleratorPrefix}Installer-Key`; @@ -70,6 +71,7 @@ async function main() { installerCmk, codebuildComputeType, pageSize, + backoff, }); } diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 2f8f66cc7..57f665d6c 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -57,6 +57,7 @@ export namespace InitialSetup { // eslint-disable-next-line @typescript-eslint/no-explicit-any codebuildComputeType: any; pageSize: string; + backoff: string | undefined; /** * Current Accelerator version */ @@ -264,6 +265,7 @@ export namespace InitialSetup { SUBNET_CIDR_ASSIGNED_POOL: subnetCidrPoolTable.tableName, CIDR_POOL: cidrPoolTable.tableName, DEPLOY_STACK_PAGE_SIZE: props.pageSize, + BACKOFF_START_DELAY: props.backoff || '', COMPUTE_TYPE: props.codebuildComputeType, }, }); diff --git a/src/deployments/cdk/src/common/alb-ip-forwarding.ts b/src/deployments/cdk/src/common/alb-ip-forwarding.ts index b79a36a2f..de0f68fab 100644 --- a/src/deployments/cdk/src/common/alb-ip-forwarding.ts +++ b/src/deployments/cdk/src/common/alb-ip-forwarding.ts @@ -59,7 +59,7 @@ export class AlbIpForwarding extends Construct { runtime: lambda.Runtime.NODEJS_14_X, code: lambdaCode, handler: 'index.albIpMonitor', - timeout: Duration.seconds(30), + timeout: Duration.seconds(60), environment: { LOOKUP_TABLE: ddbTable.tableName, }, @@ -82,7 +82,7 @@ export class AlbIpForwarding extends Construct { runtime: lambda.Runtime.NODEJS_14_X, handler: 'index.albTargetRecordMonitor', code: lambdaCode, - timeout: Duration.seconds(30), + timeout: Duration.seconds(60), environment: { LOOKUP_TABLE: ddbTable.tableName, }, diff --git a/src/installer/cdk/src/index.ts b/src/installer/cdk/src/index.ts index d86385ce3..d82c9c9e8 100644 --- a/src/installer/cdk/src/index.ts +++ b/src/installer/cdk/src/index.ts @@ -115,6 +115,12 @@ class Installer extends cdk.Stack { default: 680, }); + const backoffStartDelay = new cdk.CfnParameter(this, 'Backoff Start Delay', { + description: + 'The start delay for exponential backoff of API calls in milliseconds. Leave at the default of 2000 unless needed.', + default: 2000, + }); + const stateMachineName = `${acceleratorPrefix}MainStateMachine_sm`; // The state machine name has to match the name of the state machine in initial setup @@ -322,6 +328,10 @@ class Installer extends cdk.Stack { type: codebuild.BuildEnvironmentVariableType.PLAINTEXT, value: stackDeployPageSize.valueAsString, }, + BACKOFF_START_DELAY: { + type: codebuild.BuildEnvironmentVariableType.PLAINTEXT, + value: backoffStartDelay.valueAsString, + }, }, }, cache: codebuild.Cache.local(codebuild.LocalCacheMode.SOURCE), diff --git a/src/lib/cdk-accelerator/src/core/accelerator-stack.ts b/src/lib/cdk-accelerator/src/core/accelerator-stack.ts index a813c0218..f46dc3e40 100644 --- a/src/lib/cdk-accelerator/src/core/accelerator-stack.ts +++ b/src/lib/cdk-accelerator/src/core/accelerator-stack.ts @@ -12,7 +12,7 @@ */ import * as cdk from '@aws-cdk/core'; -import { AcceleratorNameTagger, AcceleratorProtectedTagger } from '.'; +import { AcceleratorNameTagger, AcceleratorProtectedTagger, LambdaEnvironmentVariables, LambdaDefaultTimeout } from '.'; export interface AcceleratorStackProps extends cdk.StackProps { acceleratorName: string; @@ -34,6 +34,8 @@ export class AcceleratorStack extends cdk.Stack { cdk.Aspects.of(this).add(new cdk.Tag('Accelerator', this.acceleratorName)); cdk.Aspects.of(this).add(new AcceleratorNameTagger()); cdk.Aspects.of(this).add(new AcceleratorProtectedTagger(this.acceleratorName)); + cdk.Aspects.of(this).add(new LambdaEnvironmentVariables()); + cdk.Aspects.of(this).add(new LambdaDefaultTimeout()); } static of(construct: cdk.IConstruct): AcceleratorStack { diff --git a/src/lib/cdk-accelerator/src/core/index.ts b/src/lib/cdk-accelerator/src/core/index.ts index a38054f14..e58f07144 100644 --- a/src/lib/cdk-accelerator/src/core/index.ts +++ b/src/lib/cdk-accelerator/src/core/index.ts @@ -14,3 +14,5 @@ export * from './cfn-include'; export * from './name-tagger'; export * from './accelerator-protected-tagger'; +export * from './lambda-env-variables'; +export * from './lambda-default-timeout'; diff --git a/src/lib/cdk-accelerator/src/core/lambda-default-timeout.ts b/src/lib/cdk-accelerator/src/core/lambda-default-timeout.ts new file mode 100644 index 000000000..0f8acad69 --- /dev/null +++ b/src/lib/cdk-accelerator/src/core/lambda-default-timeout.ts @@ -0,0 +1,12 @@ +import * as cdk from '@aws-cdk/core'; +import * as lambda from '@aws-cdk/aws-lambda'; + +export class LambdaDefaultTimeout implements cdk.IAspect { + visit(node: cdk.IConstruct): void { + if (node instanceof lambda.CfnFunction) { + if (!node.timeout) { + node.timeout = 60; + } + } + } +} diff --git a/src/lib/cdk-accelerator/src/core/lambda-env-variables.ts b/src/lib/cdk-accelerator/src/core/lambda-env-variables.ts new file mode 100644 index 000000000..a50674a44 --- /dev/null +++ b/src/lib/cdk-accelerator/src/core/lambda-env-variables.ts @@ -0,0 +1,10 @@ +import * as cdk from '@aws-cdk/core'; +import * as lambda from '@aws-cdk/aws-lambda'; + +export class LambdaEnvironmentVariables implements cdk.IAspect { + visit(node: cdk.IConstruct): void { + if (node instanceof lambda.Function) { + node.addEnvironment('BACKOFF_START_DELAY', process.env['BACKOFF_START_DELAY'] || '2000'); + } + } +} diff --git a/src/lib/common/src/aws/backoff.ts b/src/lib/common/src/aws/backoff.ts index bec056d47..23ee07437 100644 --- a/src/lib/common/src/aws/backoff.ts +++ b/src/lib/common/src/aws/backoff.ts @@ -20,8 +20,24 @@ export function throttlingBackOff( request: () => Promise, options?: Partial>, ): Promise { + const defaultDelay = 500; + let maxDelayValue = 2000; + + if (process.env.BACKOFF_START_DELAY) { + const backoffStartDelay = parseInt(process.env.BACKOFF_START_DELAY, 10); + if (Number.isInteger(backoffStartDelay)) { + maxDelayValue = backoffStartDelay; + } + } + + // Add jitter to the starting delay + const startingDelay = Math.random() * (maxDelayValue - defaultDelay + 1) + defaultDelay; + + console.log(`throttlingBackOff delay set to ${startingDelay}`); + return backOff(request, { - startingDelay: 500, + startingDelay, + delayFirstAttempt: true, jitter: 'full', retry: isThrottlingError, ...options, @@ -34,6 +50,7 @@ export const isThrottlingError = (e: any) => e.code === 'ConcurrentModificationException' || // Retry for AWS Organizations e.code === 'InsufficientDeliveryPolicyException' || // Retry for ConfigService e.code === 'NoAvailableDeliveryChannelException' || // Retry for ConfigService + e.code === 'ConcurrentModifications' || // Retry for AssociateHostedZone e.code === 'TooManyRequestsException' || e.code === 'Throttling' || e.code === 'ThrottlingException' || diff --git a/src/lib/custom-resources/cdk-acm-import-certificate/cdk/index.ts b/src/lib/custom-resources/cdk-acm-import-certificate/cdk/index.ts index 76352f486..ee0f1cef6 100644 --- a/src/lib/custom-resources/cdk-acm-import-certificate/cdk/index.ts +++ b/src/lib/custom-resources/cdk-acm-import-certificate/cdk/index.ts @@ -122,7 +122,7 @@ export class AcmImportCertificate extends cdk.Construct implements cdk.ITaggable code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/cdk-cfn-utils/cdk/index.ts b/src/lib/custom-resources/cdk-cfn-utils/cdk/index.ts index a4b0d75cc..2a086040f 100644 --- a/src/lib/custom-resources/cdk-cfn-utils/cdk/index.ts +++ b/src/lib/custom-resources/cdk-cfn-utils/cdk/index.ts @@ -22,8 +22,24 @@ export function throttlingBackOff( request: () => Promise, options?: Partial>, ): Promise { + const defaultDelay = 500; + let maxDelayValue = 2000; + + if (process.env.BACKOFF_START_DELAY) { + const backoffStartDelay = parseInt(process.env.BACKOFF_START_DELAY, 10); + if (Number.isInteger(backoffStartDelay)) { + maxDelayValue = backoffStartDelay; + } + } + + // Add jitter to the starting delay + const startingDelay = Math.random() * (maxDelayValue - defaultDelay + 1) + defaultDelay; + + console.log(`throttlingBackOff delay set to ${startingDelay}`); + return backOff(request, { - startingDelay: 500, + startingDelay, + delayFirstAttempt: true, jitter: 'full', retry: isThrottlingError, ...options, diff --git a/src/lib/custom-resources/cdk-cur-report-definition/cdk/index.ts b/src/lib/custom-resources/cdk-cur-report-definition/cdk/index.ts index a809da332..6bf5c75e9 100644 --- a/src/lib/custom-resources/cdk-cur-report-definition/cdk/index.ts +++ b/src/lib/custom-resources/cdk-cur-report-definition/cdk/index.ts @@ -135,7 +135,7 @@ export class CurReportDefinition extends cdk.Construct { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/cdk-ec2-ebs-default-encryption/cdk/index.ts b/src/lib/custom-resources/cdk-ec2-ebs-default-encryption/cdk/index.ts index b770dba30..ad1fcfb10 100644 --- a/src/lib/custom-resources/cdk-ec2-ebs-default-encryption/cdk/index.ts +++ b/src/lib/custom-resources/cdk-ec2-ebs-default-encryption/cdk/index.ts @@ -87,7 +87,7 @@ export class EbsDefaultEncryption extends cdk.Construct { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/cdk-ec2-keypair/cdk/index.ts b/src/lib/custom-resources/cdk-ec2-keypair/cdk/index.ts index 5bbd9aeea..ff01e8138 100644 --- a/src/lib/custom-resources/cdk-ec2-keypair/cdk/index.ts +++ b/src/lib/custom-resources/cdk-ec2-keypair/cdk/index.ts @@ -100,7 +100,7 @@ export class Keypair extends cdk.Construct implements cdk.ITaggable { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/cdk-kms-grant/cdk/index.ts b/src/lib/custom-resources/cdk-kms-grant/cdk/index.ts index dd75faeff..31122e398 100644 --- a/src/lib/custom-resources/cdk-kms-grant/cdk/index.ts +++ b/src/lib/custom-resources/cdk-kms-grant/cdk/index.ts @@ -118,7 +118,7 @@ export class Grant extends cdk.Construct { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/cdk-s3-update-logarchive-bucket-policy/cdk/index.ts b/src/lib/custom-resources/cdk-s3-update-logarchive-bucket-policy/cdk/index.ts index 056e3219a..10b806f2f 100644 --- a/src/lib/custom-resources/cdk-s3-update-logarchive-bucket-policy/cdk/index.ts +++ b/src/lib/custom-resources/cdk-s3-update-logarchive-bucket-policy/cdk/index.ts @@ -106,7 +106,7 @@ export class S3UpdateLogArchivePolicy extends cdk.Construct { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(30), + timeout: cdk.Duration.seconds(60), }); } } diff --git a/src/lib/custom-resources/ec2-ebs-default-encryption/lib/index.ts b/src/lib/custom-resources/ec2-ebs-default-encryption/lib/index.ts index b770dba30..ad1fcfb10 100644 --- a/src/lib/custom-resources/ec2-ebs-default-encryption/lib/index.ts +++ b/src/lib/custom-resources/ec2-ebs-default-encryption/lib/index.ts @@ -87,7 +87,7 @@ export class EbsDefaultEncryption extends cdk.Construct { code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', role, - timeout: cdk.Duration.seconds(10), + timeout: cdk.Duration.seconds(60), }); } }