From 0da4bfd5726f6829c5ebfa619895e5a8ee79cc0b Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 19 Aug 2020 11:40:48 +0530 Subject: [PATCH 1/8] feat(core): Adding intial Commit for CWL Centralized logging --- src/deployments/cdk/src/apps/phase-0.ts | 8 + src/deployments/cdk/src/apps/phase-1.ts | 7 +- .../central-logging-s3/outputs.ts | 5 + .../central-logging-s3/step-1.ts | 149 ++++++++---------- .../deployments/central-services/step-1.ts | 2 +- .../iam/cwl-central-logging-roles.ts | 98 ++++++++++++ .../cdk/src/deployments/iam/index.ts | 1 + src/lib/common-config/src/index.ts | 7 + src/lib/common-outputs/src/log-destination.ts | 23 +++ 9 files changed, 210 insertions(+), 90 deletions(-) create mode 100644 src/deployments/cdk/src/deployments/central-services/central-logging-s3/outputs.ts create mode 100644 src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts create mode 100644 src/lib/common-outputs/src/log-destination.ts diff --git a/src/deployments/cdk/src/apps/phase-0.ts b/src/deployments/cdk/src/apps/phase-0.ts index aad080060..526456398 100644 --- a/src/deployments/cdk/src/apps/phase-0.ts +++ b/src/deployments/cdk/src/apps/phase-0.ts @@ -167,6 +167,14 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte config: acceleratorConfig, }); + // Creating roles required for CWL Central Logging + await iamDeployment.createCwlCentralLoggingRoles({ + acceleratorPrefix: context.acceleratorPrefix, + accountStacks, + config: acceleratorConfig, + logBucket, + }); + /** * Code to create LogGroups required for DNS Logging */ diff --git a/src/deployments/cdk/src/apps/phase-1.ts b/src/deployments/cdk/src/apps/phase-1.ts index 453cf11ed..a9700ca46 100644 --- a/src/deployments/cdk/src/apps/phase-1.ts +++ b/src/deployments/cdk/src/apps/phase-1.ts @@ -433,13 +433,12 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte } // Central Services step 1 - const shardCount = acceleratorConfig['global-options']['central-log-services']['kinesis-stream-shard-count']; - const logsAccountStack = accountStacks.getOrCreateAccountStack(logAccountKey); await cwlCentralLoggingToS3.step1({ - accountStack: logsAccountStack, + accountStacks, accounts, logBucket, - shardCount, + outputs, + config: acceleratorConfig, }); await vpcDeployment.step1({ diff --git a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/outputs.ts b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/outputs.ts new file mode 100644 index 000000000..330bc09b3 --- /dev/null +++ b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/outputs.ts @@ -0,0 +1,5 @@ +import * as t from 'io-ts'; +import { createCfnStructuredOutput } from '../../../common/structured-output'; +import { LogDestinationOutput } from '@aws-accelerator/common-outputs/src/log-destination'; + +export const CfnLogDestinationOutput = createCfnStructuredOutput(LogDestinationOutput); diff --git a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-1.ts b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-1.ts index cc141647c..874255860 100644 --- a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-1.ts +++ b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-1.ts @@ -1,36 +1,77 @@ import * as cdk from '@aws-cdk/core'; -import * as iam from '@aws-cdk/aws-iam'; -import { createRoleName, createName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; +import { createName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; import * as kinesis from '@aws-cdk/aws-kinesis'; import * as s3 from '@aws-cdk/aws-s3'; import * as logs from '@aws-cdk/aws-logs'; import * as kinesisfirehose from '@aws-cdk/aws-kinesisfirehose'; -import { AccountStack } from '../../../common/account-stacks'; +import { AccountStacks } from '../../../common/account-stacks'; import { Account } from '../../../utils/accounts'; import { JsonOutputValue } from '../../../common/json-output'; import { CLOUD_WATCH_CENTRAL_LOGGING_BUCKET_PREFIX } from '@aws-accelerator/common/src/util/constants'; +import * as c from '@aws-accelerator/common-config'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; +import { CfnLogDestinationOutput } from './outputs'; export interface CentralLoggingToS3Step1Props { - accountStack: AccountStack; + accountStacks: AccountStacks; accounts: Account[]; logBucket: s3.IBucket; - shardCount?: number; + outputs: StackOutput[]; + config: c.AcceleratorConfig; } /** * Enable Central Logging to S3 in "log-archive" account Step 1 */ export async function step1(props: CentralLoggingToS3Step1Props) { - const { accountStack, accounts, logBucket, shardCount } = props; + const { accountStacks, accounts, logBucket, config, outputs } = props; // Setup for CloudWatch logs storing in logs account const allAccountIds = accounts.map(account => account.id); - await cwlSettingsInLogArchive({ - scope: accountStack, - accountIds: allAccountIds, - bucketArn: logBucket.bucketArn, - encryptionKey: logBucket.encryptionKey?.keyArn!, - shardCount, + const centralLogServices = config['global-options']['central-log-services']; + const cwlRegionsConfig = config['global-options']['additional-cwl-regions']; + if (!cwlRegionsConfig[centralLogServices.region]) { + cwlRegionsConfig[centralLogServices.region] = { + 'kinesis-stream-shard-count': centralLogServices['kinesis-stream-shard-count'], + }; + } + + const cwlLogStreamRoleOutput = IamRoleOutputFinder.tryFindOneByName({ + outputs, + accountKey: centralLogServices.account, + roleKey: 'CWLLogsStreamRole', + }); + + const cwlKinesisStreamRoleOutput = IamRoleOutputFinder.tryFindOneByName({ + outputs, + accountKey: centralLogServices.account, + roleKey: 'CWLKinesisStreamRole', }); + + if (!cwlLogStreamRoleOutput || !cwlKinesisStreamRoleOutput) { + console.error(`Skipping CWL Central logging setup due to unavailability of roles in output`); + return; + } + + // Setting up in default "central-log-services" and "additional-cwl-regions" region + for (const [region, regionConfig] of Object.entries(cwlRegionsConfig)) { + // Setup CWL Central logging in default region + const logAccountStack = accountStacks.tryGetOrCreateAccountStack(centralLogServices.account, region); + if (!logAccountStack) { + console.error( + `Cannot find account stack ${centralLogServices.account}: ${region} while setting up cloudWatch central logging to S3`, + ); + continue; + } + await cwlSettingsInLogArchive({ + scope: logAccountStack, + accountIds: allAccountIds, + bucketArn: logBucket.bucketArn, + shardCount: regionConfig['kinesis-stream-shard-count'], + logStreamRoleArn: cwlLogStreamRoleOutput.roleArn, + kinesisStreamRoleArn: cwlKinesisStreamRoleOutput.roleArn, + }); + } } /** @@ -41,10 +82,11 @@ async function cwlSettingsInLogArchive(props: { scope: cdk.Construct; accountIds: string[]; bucketArn: string; - encryptionKey: string; + logStreamRoleArn: string; + kinesisStreamRoleArn: string; shardCount?: number; }) { - const { scope, accountIds, bucketArn, encryptionKey, shardCount } = props; + const { scope, accountIds, bucketArn, logStreamRoleArn, kinesisStreamRoleArn, shardCount } = props; // Create Kinesis Stream for Logs streaming const logsStream = new kinesis.Stream(scope, 'Logs-Stream', { @@ -56,27 +98,6 @@ async function cwlSettingsInLogArchive(props: { shardCount, }); - // Create IAM Role for reading logs from stream and push to destination - const logsRole = new iam.Role(scope, 'CloudWatch-Logs-Stream-Role', { - roleName: createRoleName('CWL-Logs-Stream-Role'), - assumedBy: new iam.ServicePrincipal('logs.amazonaws.com'), - }); - - // Create IAM Policy for reading logs from stream and push to destination - const logsRolePolicy = new iam.Policy(scope, 'CWL-Logs-Stream-Policy', { - roles: [logsRole], - statements: [ - new iam.PolicyStatement({ - resources: [logsStream.streamArn], - actions: ['kinesis:PutRecord'], - }), - new iam.PolicyStatement({ - resources: [logsRole.roleArn], - actions: ['iam:PassRole'], - }), - ], - }); - const destinationName = createName({ name: 'LogDestination', suffixLength: 0, @@ -100,57 +121,18 @@ async function cwlSettingsInLogArchive(props: { const logDestination = new logs.CfnDestination(scope, 'Log-Destination', { destinationName, targetArn: logsStream.streamArn, - roleArn: logsRole.roleArn, + roleArn: logStreamRoleArn, destinationPolicy: destinationPolicyStr, }); - logDestination.node.addDependency(logsRolePolicy); - - // Creating IAM role for Kinesis Delivery Stream Role - const kinesisStreamRole = new iam.Role(scope, 'CWL-Kinesis-Stream-Role', { - roleName: createRoleName('Kinesis-Stream-Role'), - assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), - }); - const kinesisStreamPolicy = new iam.Policy(scope, 'CWL-Kinesis-Stream-Policy', { - roles: [kinesisStreamRole], - statements: [ - new iam.PolicyStatement({ - resources: [encryptionKey], - actions: ['kms:DescribeKey', 'kms:GenerateDataKey*', 'kms:Decrypt', 'kms:Encrypt', 'kms:ReEncrypt*'], - }), - new iam.PolicyStatement({ - resources: [bucketArn, `${bucketArn}/*`], - actions: [ - 's3:PutObject', - 's3:PutObjectAcl', - 's3:GetEncryptionConfiguration', - 's3:AbortMultipartUpload', - 's3:GetBucketLocation', - 's3:GetObject', - 's3:ListBucket', - 's3:ListBucketMultipartUploads', - 's3:PutObject', - ], - }), - new iam.PolicyStatement({ - resources: ['*'], - actions: ['kinesis:DescribeStream', 'kinesis:GetShardIterator', 'kinesis:GetRecords', 'kinesis:ListShards'], - }), - new iam.PolicyStatement({ - resources: ['arn:aws:logs:*:*:*'], - actions: ['logs:PutLogEvents'], - }), - ], - }); - - const kinesisDeliveryStream = new kinesisfirehose.CfnDeliveryStream(scope, 'Kinesis-Firehouse-Stream', { + new kinesisfirehose.CfnDeliveryStream(scope, 'Kinesis-Firehouse-Stream', { deliveryStreamName: createName({ name: 'Kinesis-Delivery-Stream', }), deliveryStreamType: 'KinesisStreamAsSource', kinesisStreamSourceConfiguration: { kinesisStreamArn: logsStream.streamArn, - roleArn: kinesisStreamRole.roleArn, + roleArn: kinesisStreamRoleArn, }, extendedS3DestinationConfiguration: { bucketArn, @@ -159,18 +141,15 @@ async function cwlSettingsInLogArchive(props: { sizeInMBs: 50, }, compressionFormat: 'UNCOMPRESSED', - roleArn: kinesisStreamRole.roleArn, + roleArn: kinesisStreamRoleArn, prefix: CLOUD_WATCH_CENTRAL_LOGGING_BUCKET_PREFIX, }, }); - kinesisDeliveryStream.node.addDependency(kinesisStreamPolicy); - kinesisDeliveryStream.node.addDependency(logsRolePolicy); // Store LogDestination ARN in output so that subsequent phases can access the output - new JsonOutputValue(scope, `CloudWatchCentralLoggingOutput`, { - type: 'CloudWatchCentralLogging', - value: { - logDestination: logDestination.attrArn, - }, + new CfnLogDestinationOutput(scope, `CloudWatchCentralLoggingOutput`, { + destinationArn: logDestination.attrArn, + destinationName: logDestination.destinationName, + destinationKey: 'CwlCentralLogDestination', }); } diff --git a/src/deployments/cdk/src/deployments/central-services/step-1.ts b/src/deployments/cdk/src/deployments/central-services/step-1.ts index 1de08efb7..820282713 100644 --- a/src/deployments/cdk/src/deployments/central-services/step-1.ts +++ b/src/deployments/cdk/src/deployments/central-services/step-1.ts @@ -1,5 +1,5 @@ import * as cdk from '@aws-cdk/core'; -import * as c from '@aws-accelerator/common-config/src'; +import * as c from '@aws-accelerator/common-config'; import { AccountStacks } from '../../common/account-stacks'; import { Account, getAccountId } from '../../utils/accounts'; import * as iam from '@aws-cdk/aws-iam'; diff --git a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts new file mode 100644 index 000000000..8031f4a1e --- /dev/null +++ b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts @@ -0,0 +1,98 @@ +import * as c from '@aws-accelerator/common-config/src'; +import * as iam from '@aws-cdk/aws-iam'; +import { AccountStacks } from '../../common/account-stacks'; +import { createRoleName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; +import { CfnIamRoleOutput } from './outputs'; +import * as s3 from '@aws-cdk/aws-s3'; + +export interface CwlCentralLoggingRoleProps { + acceleratorPrefix: string; + accountStacks: AccountStacks; + config: c.AcceleratorConfig; + logBucket: s3.IBucket; +} + +export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleProps): Promise { + const { accountStacks, config, acceleratorPrefix, logBucket } = props; + const centralLoggingServices = config['global-options']['central-log-services']; + + const accountStack = accountStacks.tryGetOrCreateAccountStack( + centralLoggingServices.account, + centralLoggingServices.region, + ); + if (!accountStack) { + throw new Error( + `Not able to create stack for "${centralLoggingServices.account}" whicle creating roles for CWL Central logging`, + ); + } + // Create IAM Role for reading logs from stream and push to destination + const logsRole = new iam.Role(accountStack, 'CloudWatch-Logs-Stream-Role', { + roleName: createRoleName('CWL-Logs-Stream-Role'), + assumedBy: new iam.ServicePrincipal('logs.amazonaws.com'), + }); + + // Create IAM Policy for reading logs from stream and push to destination + new iam.Policy(accountStack, 'CWL-Logs-Stream-Policy', { + roles: [logsRole], + statements: [ + new iam.PolicyStatement({ + resources: [`arn:aws:kinesis:*:${accountStack.accountId}:stream/${acceleratorPrefix}*`], + actions: ['kinesis:PutRecord'], + }), + new iam.PolicyStatement({ + resources: [logsRole.roleArn], + actions: ['iam:PassRole'], + }), + ], + }); + + // Creating IAM role for Kinesis Delivery Stream Role + const kinesisStreamRole = new iam.Role(accountStack, 'CWL-Kinesis-Stream-Role', { + roleName: createRoleName('Kinesis-Stream-Role'), + assumedBy: new iam.ServicePrincipal('firehose.amazonaws.com'), + }); + + new iam.Policy(accountStack, 'CWL-Kinesis-Stream-Policy', { + roles: [kinesisStreamRole], + statements: [ + new iam.PolicyStatement({ + resources: [logBucket.encryptionKey?.keyArn!], + actions: ['kms:DescribeKey', 'kms:GenerateDataKey*', 'kms:Decrypt', 'kms:Encrypt', 'kms:ReEncrypt*'], + }), + new iam.PolicyStatement({ + resources: [logBucket.bucketArn, `${logBucket.bucketArn}/*`], + actions: [ + 's3:PutObject', + 's3:PutObjectAcl', + 's3:GetEncryptionConfiguration', + 's3:AbortMultipartUpload', + 's3:GetBucketLocation', + 's3:GetObject', + 's3:ListBucket', + 's3:ListBucketMultipartUploads', + 's3:PutObject', + ], + }), + new iam.PolicyStatement({ + resources: ['*'], + actions: ['kinesis:DescribeStream', 'kinesis:GetShardIterator', 'kinesis:GetRecords', 'kinesis:ListShards'], + }), + new iam.PolicyStatement({ + resources: ['arn:aws:logs:*:*:*'], + actions: ['logs:PutLogEvents'], + }), + ], + }); + + new CfnIamRoleOutput(accountStack, `CWLLogsStreamRoleOutput`, { + roleName: logsRole.roleName, + roleArn: logsRole.roleArn, + roleKey: 'CWLLogsStreamRole', + }); + + new CfnIamRoleOutput(accountStack, `CWLKinesisStreamRoleOutput`, { + roleName: kinesisStreamRole.roleName, + roleArn: kinesisStreamRole.roleArn, + roleKey: 'CWLKinesisStreamRole', + }); +} diff --git a/src/deployments/cdk/src/deployments/iam/index.ts b/src/deployments/cdk/src/deployments/iam/index.ts index 754494238..b7cbbd4e4 100644 --- a/src/deployments/cdk/src/deployments/iam/index.ts +++ b/src/deployments/cdk/src/deployments/iam/index.ts @@ -7,3 +7,4 @@ export * from './securityhub-roles'; export * from './create-role'; export * from './ssm-document-roles'; export * from './log-group-role'; +export * from './cwl-central-logging-roles'; diff --git a/src/lib/common-config/src/index.ts b/src/lib/common-config/src/index.ts index 78eddc004..de887ec6e 100644 --- a/src/lib/common-config/src/index.ts +++ b/src/lib/common-config/src/index.ts @@ -645,6 +645,12 @@ export const VpcFlowLogsConfigType = t.interface({ }); export type VpcFlowLogsConfig = t.TypeOf; +export const AdditionalCwlRegionType = t.interface({ + 'kinesis-stream-shard-count': optional(t.number), +}); + +export type AdditionalCwlRegion = t.TypeOf; + export const GlobalOptionsConfigType = t.interface({ 'alz-baseline': t.boolean, 'ct-baseline': t.boolean, @@ -670,6 +676,7 @@ export const GlobalOptionsConfigType = t.interface({ 'workloadaccounts-suffix': optional(t.number), 'workloadaccounts-param-filename': t.string, 'vpc-flow-logs': VpcFlowLogsConfigType, + 'additional-cwl-regions': fromNullable(t.record(t.string, AdditionalCwlRegionType), {}), }); export type CentralServicesConfig = t.TypeOf; diff --git a/src/lib/common-outputs/src/log-destination.ts b/src/lib/common-outputs/src/log-destination.ts new file mode 100644 index 000000000..e659ce3e1 --- /dev/null +++ b/src/lib/common-outputs/src/log-destination.ts @@ -0,0 +1,23 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; +import { StackOutput } from './stack-output'; + +export const LogDestinationOutput = t.interface( + { + destinationName: t.string, + destinationArn: t.string, + destinationKey: t.string, + }, + 'LogDestination', +); + +export type LogDestinationOutput = t.TypeOf; + +export const LogDestinationOutputFinder = createStructuredOutputFinder(LogDestinationOutput, finder => ({ + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; destinationKey?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + predicate: o => o.destinationKey === props.destinationKey, + }), +})); From 390c33d46939e875e9533560a40e6066579c2931 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 19 Aug 2020 13:22:22 +0530 Subject: [PATCH 2/8] Adding all supported regions to assumedBy of LogStream Role --- .../cdk/src/deployments/iam/cwl-central-logging-roles.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts index 8031f4a1e..486b4fc16 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts @@ -16,6 +16,11 @@ export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleP const { accountStacks, config, acceleratorPrefix, logBucket } = props; const centralLoggingServices = config['global-options']['central-log-services']; + const cwlRegions = Object.entries(config['global-options']["additional-cwl-regions"]).map(([region, _]) => region); + if (!cwlRegions.includes(centralLoggingServices.region)) { + cwlRegions.push(centralLoggingServices.region); + } + const accountStack = accountStacks.tryGetOrCreateAccountStack( centralLoggingServices.account, centralLoggingServices.region, @@ -28,7 +33,9 @@ export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleP // Create IAM Role for reading logs from stream and push to destination const logsRole = new iam.Role(accountStack, 'CloudWatch-Logs-Stream-Role', { roleName: createRoleName('CWL-Logs-Stream-Role'), - assumedBy: new iam.ServicePrincipal('logs.amazonaws.com'), + assumedBy: new iam.CompositePrincipal( + ...cwlRegions.map(r => new iam.ServicePrincipal(`logs.${r}.amazonaws.com`)) + ), }); // Create IAM Policy for reading logs from stream and push to destination From 11d48a5761806b4f37653c71ead331d5c40c2bee Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 19 Aug 2020 13:22:39 +0530 Subject: [PATCH 3/8] prettier --- .../cdk/src/deployments/iam/cwl-central-logging-roles.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts index 486b4fc16..ef1209405 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts @@ -16,7 +16,7 @@ export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleP const { accountStacks, config, acceleratorPrefix, logBucket } = props; const centralLoggingServices = config['global-options']['central-log-services']; - const cwlRegions = Object.entries(config['global-options']["additional-cwl-regions"]).map(([region, _]) => region); + const cwlRegions = Object.entries(config['global-options']['additional-cwl-regions']).map(([region, _]) => region); if (!cwlRegions.includes(centralLoggingServices.region)) { cwlRegions.push(centralLoggingServices.region); } @@ -33,9 +33,7 @@ export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleP // Create IAM Role for reading logs from stream and push to destination const logsRole = new iam.Role(accountStack, 'CloudWatch-Logs-Stream-Role', { roleName: createRoleName('CWL-Logs-Stream-Role'), - assumedBy: new iam.CompositePrincipal( - ...cwlRegions.map(r => new iam.ServicePrincipal(`logs.${r}.amazonaws.com`)) - ), + assumedBy: new iam.CompositePrincipal(...cwlRegions.map(r => new iam.ServicePrincipal(`logs.${r}.amazonaws.com`))), }); // Create IAM Policy for reading logs from stream and push to destination From 03c5725cf0c9b6cedda537f3e3b239a78c7eff2c Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 19 Aug 2020 15:24:51 +0530 Subject: [PATCH 4/8] Creating roles required for subscription in phase -1 and adding subscription filter in additional regions --- src/deployments/cdk/src/apps/phase--1.ts | 13 ++++- .../central-logging-s3/step-2.ts | 56 +++++++++++++------ .../iam/cwl-add-subscription-filter-role.ts | 43 ++++++++++++++ .../cdk/src/deployments/iam/index.ts | 1 + src/lib/common-outputs/src/log-destination.ts | 3 +- .../logs-add-subscription-filter/src/index.ts | 20 ++----- 6 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts diff --git a/src/deployments/cdk/src/apps/phase--1.ts b/src/deployments/cdk/src/apps/phase--1.ts index bb9bdda66..a9281716d 100644 --- a/src/deployments/cdk/src/apps/phase--1.ts +++ b/src/deployments/cdk/src/apps/phase--1.ts @@ -9,6 +9,9 @@ import * as customResourceRoles from '../deployments/iam'; * - Creating required roles for guardDuty custom resources * - Creating required roles for securityHub custom resources * - Creating required roles for IamCreateRole custom resource + * - Creating required roles for createSSMDocument custom resource + * - Creating required roles for createLogGroup custom resource + * - Creating required roles for CWLCentralLoggingSubscriptionFilterRole custom resource */ export async function deploy({ acceleratorConfig, accountStacks, accounts }: PhaseInput) { // creates roles for macie custom resources @@ -35,15 +38,23 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts }: Pha accounts, }); - // Creates roles for IamCreateRole custom resource + // Creates roles for createSSMDocument custom resource await customResourceRoles.createSSMDocumentRoles({ accountStacks, accounts, config: acceleratorConfig, }); + // Creates roles for createLogGroup custom resource await customResourceRoles.createLogGroupRole({ accountStacks, accounts, }); + + // Creates roles for createCwlSubscriptionFilter custom resource + await customResourceRoles.createCwlAddSubscriptionFilterRoles({ + accountStacks, + accounts, + config: acceleratorConfig, + }); } diff --git a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts index 45bc7a1fa..7d82c4f92 100644 --- a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts +++ b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts @@ -1,9 +1,10 @@ import * as c from '@aws-accelerator/common-config/src'; import { AccountStacks } from '../../../common/account-stacks'; -import { Account } from '../../../utils/accounts'; import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { CentralLoggingSubscriptionFilter } from '@aws-accelerator/custom-resource-logs-add-subscription-filter'; import { createName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; +import { LogDestinationOutputFinder } from '@aws-accelerator/common-outputs/src/log-destination'; +import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; export interface CentralLoggingToS3Step2Props { accountStacks: AccountStacks; @@ -22,37 +23,56 @@ export async function step2(props: CentralLoggingToS3Step2Props) { const globalOptionsConfig = config['global-options']; const defaultLogRetention = globalOptionsConfig['default-cwl-retention']; const accountConfigs = config.getAccountConfigs(); - const logConfig = globalOptionsConfig['central-log-services']; - const logArchiveAccountKey = logConfig.account; - const logDestinationOutput = getStackJsonOutput(outputs, { - accountKey: logArchiveAccountKey, - outputType: 'CloudWatchCentralLogging', - }); - if (!logDestinationOutput || logDestinationOutput.length === 0) { - console.log(`Log Dstination not found in outputs ${logArchiveAccountKey}`); - return; + const centralLogServices = globalOptionsConfig['central-log-services']; + const cwlRegions = Object.entries(globalOptionsConfig['additional-cwl-regions']).map(([region, _]) => region); + if (!cwlRegions.includes(centralLogServices.region)) { + cwlRegions.push(centralLogServices.region); } - const logDestinationArn = logDestinationOutput[0].logDestination; + for (const [accountKey, accountConfig] of accountConfigs) { const logRetention = accountConfig['cwl-retention'] || defaultLogRetention; - const accountStack = accountStacks.tryGetOrCreateAccountStack(accountKey); - if (!accountStack) { - console.warn(`Cannot find account stack ${accountKey}`); - } else { + const subscriptionRoleOutput = IamRoleOutputFinder.tryFindOneByName({ + accountKey, + outputs, + roleKey: 'CWLAddSubscriptionFilter', + }); + if (!subscriptionRoleOutput) { + console.error(`Can't find "CWLAddSubscriptionFilter" Role in account ${accountKey} outputs`); + continue; + } + for (const region of cwlRegions) { + const logDestinationOutput = LogDestinationOutputFinder.tryFindOneByName({ + outputs, + accountKey: centralLogServices.account, + region, + destinationKey: 'CwlCentralLogDestination', + }); + if (!logDestinationOutput) { + console.warn(`Cannot find required LogDestination in account "${accountKey}"`); + return; + } + const accountStack = accountStacks.tryGetOrCreateAccountStack(accountKey, region); + if (!accountStack) { + console.error( + `Cannot find account stack ${accountKey}:${region} while adding subscription filter for CWL Central logging`, + ); + continue; + } const accountSpecificExclusions = [ - ...(logConfig['cwl-exclusions']?.find(e => e.account === accountKey)?.exclusions || []), + ...(centralLogServices['cwl-exclusions']?.find(e => e.account === accountKey)?.exclusions || []), ]; - const globalExclusions = [...(logConfig['cwl-glbl-exclusions'] || []), ...accountSpecificExclusions]; + const globalExclusions = [...(centralLogServices['cwl-glbl-exclusions'] || []), ...accountSpecificExclusions]; const ruleName = createName({ name: 'NewLogGroup_rule', account: false, region: false, }); new CentralLoggingSubscriptionFilter(accountStack, `CentralLoggingSubscriptionFilter-${accountKey}`, { - logDestinationArn, + logDestinationArn: logDestinationOutput.destinationArn, globalExclusions, ruleName, logRetention, + roleArn: subscriptionRoleOutput.roleArn, }); } } diff --git a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts new file mode 100644 index 000000000..ffffc9d29 --- /dev/null +++ b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts @@ -0,0 +1,43 @@ +import * as c from '@aws-accelerator/common-config/src'; +import * as iam from '@aws-cdk/aws-iam'; +import { AccountStacks } from '../../common/account-stacks'; +import { createRoleName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; +import { CfnIamRoleOutput } from './outputs'; +import { Account } from '../../utils/accounts'; + +export interface CwlAddSubscriptionFilterRoleProps { + accountStacks: AccountStacks; + config: c.AcceleratorConfig; + accounts: Account[]; +} + +export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscriptionFilterRoleProps): Promise { + const { accountStacks, config, accounts } = props; + const centralLoggingServices = config['global-options']['central-log-services']; + for (const account of accounts) { + const accountStack = accountStacks.tryGetOrCreateAccountStack(account.key, centralLoggingServices.region); + if (!accountStack) { + throw new Error( + `Not able to create stack for "${centralLoggingServices.account}" whicle creating roles for CWL Central logging`, + ); + } + // Create IAM Role for reading logs from stream and push to destination + const role = new iam.Role(accountStack, 'CWLAddSubscriptionFilterRole', { + roleName: createRoleName('CWL-Add-Subscription-Filter'), + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + actions: ['logs:*'], + resources: ['*'], + }), + ); + + new CfnIamRoleOutput(accountStack, `CWLLogsStreamRoleOutput`, { + roleName: role.roleName, + roleArn: role.roleArn, + roleKey: 'CWLAddSubscriptionFilter', + }); + } +} diff --git a/src/deployments/cdk/src/deployments/iam/index.ts b/src/deployments/cdk/src/deployments/iam/index.ts index b7cbbd4e4..2ab1f9414 100644 --- a/src/deployments/cdk/src/deployments/iam/index.ts +++ b/src/deployments/cdk/src/deployments/iam/index.ts @@ -8,3 +8,4 @@ export * from './create-role'; export * from './ssm-document-roles'; export * from './log-group-role'; export * from './cwl-central-logging-roles'; +export * from './cwl-add-subscription-filter-role'; diff --git a/src/lib/common-outputs/src/log-destination.ts b/src/lib/common-outputs/src/log-destination.ts index e659ce3e1..bb9419c51 100644 --- a/src/lib/common-outputs/src/log-destination.ts +++ b/src/lib/common-outputs/src/log-destination.ts @@ -14,10 +14,11 @@ export const LogDestinationOutput = t.interface( export type LogDestinationOutput = t.TypeOf; export const LogDestinationOutputFinder = createStructuredOutputFinder(LogDestinationOutput, finder => ({ - tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; destinationKey?: string }) => + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string; destinationKey?: string }) => finder.tryFindOne({ outputs: props.outputs, accountKey: props.accountKey, + region: props.region, predicate: o => o.destinationKey === props.destinationKey, }), })); diff --git a/src/lib/custom-resources/logs-add-subscription-filter/src/index.ts b/src/lib/custom-resources/logs-add-subscription-filter/src/index.ts index 9f85d3569..6649fd323 100644 --- a/src/lib/custom-resources/logs-add-subscription-filter/src/index.ts +++ b/src/lib/custom-resources/logs-add-subscription-filter/src/index.ts @@ -11,6 +11,7 @@ export interface CentralLoggingSubscriptionFilterProps { globalExclusions?: string[]; ruleName: string; logRetention: number; + roleArn: string; } /** @@ -18,6 +19,7 @@ export interface CentralLoggingSubscriptionFilterProps { */ export class CentralLoggingSubscriptionFilter extends cdk.Construct { private readonly resource: cdk.CustomResource; + private readonly role: iam.IRole; private readonly cloudWatchEnventLambdaPath = '@aws-accelerator/custom-resource-logs-add-subscription-filter-cloudwatch-event-runtime'; private readonly cloudFormationCustomLambaPath = @@ -26,6 +28,7 @@ export class CentralLoggingSubscriptionFilter extends cdk.Construct { constructor(scope: cdk.Construct, id: string, props: CentralLoggingSubscriptionFilterProps) { super(scope, id); + this.role = iam.Role.fromRoleArn(this, `${resourceType}Role`, props.roleArn); // Custom Resource to add subscriptin filter to existing logGroups this.resource = new cdk.CustomResource(this, 'Resource', { resourceType, @@ -79,10 +82,6 @@ export class CentralLoggingSubscriptionFilter extends cdk.Construct { return this.ensureLambdaFunction(this.cloudFormationCustomLambaPath, resourceType); } - get role(): iam.IRole { - return this.lambdaFunction.role!; - } - private ensureLambdaFunction( lambdaLocation: string, name: string, @@ -98,22 +97,11 @@ export class CentralLoggingSubscriptionFilter extends cdk.Construct { const lambdaPath = require.resolve(lambdaLocation); const lambdaDir = path.dirname(lambdaPath); - const role = new iam.Role(stack, `${name}Role`, { - assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), - }); - - role.addToPrincipalPolicy( - new iam.PolicyStatement({ - actions: ['logs:*'], - resources: ['*'], - }), - ); - return new lambda.Function(stack, constructName, { runtime: lambda.Runtime.NODEJS_12_X, code: lambda.Code.fromAsset(lambdaDir), handler: 'index.handler', - role, + role: this.role, environment: environment!, // Set timeout to maximum timeout timeout: cdk.Duration.minutes(15), From 8b565d69d19a805f2b4b5561f174320373de8661 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 19 Aug 2020 16:44:37 +0530 Subject: [PATCH 5/8] Changing Construct name for Role --- .../cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts index ffffc9d29..ef18bc4fe 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts @@ -34,7 +34,7 @@ export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscript }), ); - new CfnIamRoleOutput(accountStack, `CWLLogsStreamRoleOutput`, { + new CfnIamRoleOutput(accountStack, `CWLAddSubscriptionFilterRoleOutput`, { roleName: role.roleName, roleArn: role.roleArn, roleKey: 'CWLAddSubscriptionFilter', From 09254168bb2af5fc760e515b14af451fb457788c Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 21 Aug 2020 15:50:25 +0530 Subject: [PATCH 6/8] Updates based on review comments --- .../central-logging-s3/step-2.ts | 4 ++-- .../iam/cwl-add-subscription-filter-role.ts | 17 ++++++++++++++--- .../iam/cwl-central-logging-roles.ts | 5 +++-- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts index 7d82c4f92..31dc8db3f 100644 --- a/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts +++ b/src/deployments/cdk/src/deployments/central-services/central-logging-s3/step-2.ts @@ -48,8 +48,8 @@ export async function step2(props: CentralLoggingToS3Step2Props) { destinationKey: 'CwlCentralLogDestination', }); if (!logDestinationOutput) { - console.warn(`Cannot find required LogDestination in account "${accountKey}"`); - return; + console.warn(`Cannot find required LogDestination in account "${accountKey}:${region}"`); + continue; } const accountStack = accountStacks.tryGetOrCreateAccountStack(accountKey, region); if (!accountStack) { diff --git a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts index ef18bc4fe..ffef01e84 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts @@ -17,9 +17,10 @@ export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscript for (const account of accounts) { const accountStack = accountStacks.tryGetOrCreateAccountStack(account.key, centralLoggingServices.region); if (!accountStack) { - throw new Error( - `Not able to create stack for "${centralLoggingServices.account}" whicle creating roles for CWL Central logging`, + console.error( + `Not able to create stack for "${centralLoggingServices.account}" while creating role for CWL Central logging`, ); + continue; } // Create IAM Role for reading logs from stream and push to destination const role = new iam.Role(accountStack, 'CWLAddSubscriptionFilterRole', { @@ -30,7 +31,17 @@ export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscript role.addToPrincipalPolicy( new iam.PolicyStatement({ actions: ['logs:*'], - resources: ['*'], + resources: [ + 'logs:DeleteSubscriptionFilter', + 'logs:DescribeLogGroups', + 'logs:DescribeSubscriptionFilters', + 'logs:PutSubscriptionFilter', + 'logs:PutRetentionPolicy', + 'logs:CreateLogGroup', + 'logs:CreateLogStream', + 'logs:PutLogEvents', + 'logs:DeleteRetentionPolicy', + ], }), ); diff --git a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts index ef1209405..dfb16854b 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-central-logging-roles.ts @@ -26,9 +26,10 @@ export async function createCwlCentralLoggingRoles(props: CwlCentralLoggingRoleP centralLoggingServices.region, ); if (!accountStack) { - throw new Error( - `Not able to create stack for "${centralLoggingServices.account}" whicle creating roles for CWL Central logging`, + console.error( + `Not able to create stack for "${centralLoggingServices.account}:${centralLoggingServices.region}" whicle creating role for CWL Central logging`, ); + return; } // Create IAM Role for reading logs from stream and push to destination const logsRole = new iam.Role(accountStack, 'CloudWatch-Logs-Stream-Role', { From c4a29d7e2df85e121de68d6736f68bc6191ad576 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 21 Aug 2020 17:35:25 +0530 Subject: [PATCH 7/8] Changing actions --- .../src/deployments/iam/cwl-add-subscription-filter-role.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts index ffef01e84..633d00029 100644 --- a/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts +++ b/src/deployments/cdk/src/deployments/iam/cwl-add-subscription-filter-role.ts @@ -30,8 +30,7 @@ export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscript role.addToPrincipalPolicy( new iam.PolicyStatement({ - actions: ['logs:*'], - resources: [ + actions: [ 'logs:DeleteSubscriptionFilter', 'logs:DescribeLogGroups', 'logs:DescribeSubscriptionFilters', @@ -42,6 +41,7 @@ export async function createCwlAddSubscriptionFilterRoles(props: CwlAddSubscript 'logs:PutLogEvents', 'logs:DeleteRetentionPolicy', ], + resources: ['*'], }), ); From 7979135b6724b12039b4a142371a4614a4a8da1c Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 21 Aug 2020 18:11:44 +0530 Subject: [PATCH 8/8] Updating Configuration --- reference-artifacts/config.example.json | 3 ++- .../master-config-sample-snippets/sample_snippets.json | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/reference-artifacts/config.example.json b/reference-artifacts/config.example.json index 74681fd3d..95d0ee63a 100644 --- a/reference-artifacts/config.example.json +++ b/reference-artifacts/config.example.json @@ -11,7 +11,7 @@ "workloadaccounts-suffix" : 1, "workloadaccounts-prefix" : "config", "workloadaccounts-param-filename": "config.json", - "ignored-ous": [], + "ignored-ous": [], "supported-regions": [ "ap-northeast-1", "ap-northeast-2", @@ -64,6 +64,7 @@ "ssm-to-s3": true, "ssm-to-cwl": true }, + "additional-cwl-regions": {}, "reports": { "cost-and-usage-report": { "additional-schema-elements": ["RESOURCES"], diff --git a/reference-artifacts/master-config-sample-snippets/sample_snippets.json b/reference-artifacts/master-config-sample-snippets/sample_snippets.json index 94a6509f7..739dec940 100644 --- a/reference-artifacts/master-config-sample-snippets/sample_snippets.json +++ b/reference-artifacts/master-config-sample-snippets/sample_snippets.json @@ -16,6 +16,12 @@ Config File Examples: {Not in sample config file} "exclusions": ["def/*"] } ], +************************************Additional regions for Amazon CloudWatch Central Logging to S3 + "additional-cwl-regions": { + "us-east-1": { + "kinesis-stream-shard-count": 1 + } + }, ************************************cert REQUEST format "certificates": [ {