From 6b9bb4d47c3e80defbe6de747636f5edf1c71c3d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 18 Sep 2020 17:08:46 +0530 Subject: [PATCH 01/25] Initial Push with starter code --- src/core/runtime/src/index.ts | 1 + .../runtime/src/save-outputs-to-ssm/index.ts | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/index.ts diff --git a/src/core/runtime/src/index.ts b/src/core/runtime/src/index.ts index 20c6baa23..d331ce840 100644 --- a/src/core/runtime/src/index.ts +++ b/src/core/runtime/src/index.ts @@ -23,6 +23,7 @@ export { handler as verifyFilesStep } from './verify-files-step'; export { handler as notifySMFailure } from './notify-statemachine-failure'; export { handler as notifySMSuccess } from './notify-statemachine-success'; export { handler as getAccountInfo } from './get-account-info'; +export { handler as saveOutputsToSSM } from './save-outputs-to-ssm'; // TODO Replace with // export * as codebuild from './codebuild'; diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts new file mode 100644 index 000000000..683696494 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -0,0 +1,33 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; +import { LoadConfigurationInput } from '../load-configuration-step'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; + +export interface SaveOutputsToSsmInput extends LoadConfigurationInput { + acceleratorPrefix: string; + account: Account; + region: string; + outputTableName: string; + outputType: string; +} + +const dynamodb = new DynamoDB(); + +export const handler = async (input: SaveOutputsToSsmInput) => { + console.log(`Adding service control policy to organization...`); + console.log(JSON.stringify(input, null, 2)); + + const { acceleratorPrefix, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; + + // Retrieve Configuration from Code Commit with specific commitId + const config = await loadAcceleratorConfig({ + repositoryName: configRepositoryName, + filePath: configFilePath, + commitId: configCommitId, + }); + + return { + status: 'SUCCESS', + }; +}; From 74326d342271670c6fa28039a45204cdc6257483 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 21 Sep 2020 11:04:27 +0530 Subject: [PATCH 02/25] Initial Push for Network outputs --- .../runtime/src/save-outputs-to-ssm/index.ts | 27 ++++++++++++++++--- .../save-outputs-to-ssm/network-outputs.ts | 27 +++++++++++++++++++ .../runtime/src/save-outputs-to-ssm/utils.ts | 12 +++++++++ src/lib/common/src/aws/dynamodb.ts | 12 +++++++++ 4 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts create mode 100644 src/core/runtime/src/save-outputs-to-ssm/utils.ts diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 683696494..b4922b3ed 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -3,13 +3,13 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { saveNetworkOutputs } from './network-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; account: Account; region: string; - outputTableName: string; - outputType: string; + outputsTableName: string; } const dynamodb = new DynamoDB(); @@ -18,7 +18,7 @@ export const handler = async (input: SaveOutputsToSsmInput) => { console.log(`Adding service control policy to organization...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; + const { acceleratorPrefix, configRepositoryName, configFilePath, configCommitId, outputsTableName, account } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -27,7 +27,28 @@ export const handler = async (input: SaveOutputsToSsmInput) => { commitId: configCommitId, }); + // Store Network Outputs to SSM Parameter Store + await saveNetworkOutputs(outputsTableName, dynamodb, config, account); + return { status: 'SUCCESS', }; }; + +handler({ + "acceleratorPrefix": "PBMMAccel-", + "outputsTableName": "PBMMAccel-Outputs", + "region": "eu-west-2", + "account": { + "arn": "arn:aws:organizations::538235518685:account/o-wdw2wt4bk9/233932606131", + "email": "nkoppula+non-alz-5-security@amazon.com", + "id": "233932606131", + "key": "security", + "name": "security", + "ou": "corerename", + "ouPath": "corerename" + }, + configCommitId: 'master', + configFilePath: 'raw/config.json', + configRepositoryName: 'PBMMAccel-Config-Repo' +}); diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts new file mode 100644 index 000000000..50aa0db04 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -0,0 +1,27 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { AcceleratorConfig } from '@aws-accelerator/common-config'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput } from './utils'; + +/** + * Outputs for network related deployments will be found in following phases + * + */ + + +/** + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveNetworkOutputs(outputsTableName: string, dynamodb: DynamoDB, config: AcceleratorConfig, account: Account) { + const vpcConfigs = config.getVpcConfigs(); + for (const {accountKey, vpcConfig, ouKey} of vpcConfigs) { + const outputs: StackOutput[] = await getOutput(outputsTableName, `${accountKey}-${vpcConfig.region}-1`, dynamodb); + } +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts new file mode 100644 index 000000000..349088fc8 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -0,0 +1,12 @@ +import { StackOutput } from "@aws-accelerator/common-outputs/src/stack-output" +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; + +export async function getOutput(tableName: string, key: string, dynamodb: DynamoDB): Promise { + const outputs: StackOutput[] = []; + const cfnOutputs = await dynamodb.getOutputValue(tableName, key); + if (!cfnOutputs || !cfnOutputs.S) { + return outputs; + } + outputs.push(...JSON.parse(cfnOutputs.S)); + return outputs; +} \ No newline at end of file diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 3f110d200..e5098b00b 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -60,4 +60,16 @@ export class DynamoDB { async updateItem(props: dynamodb.UpdateItemInput): Promise { await throttlingBackOff(() => this.client.updateItem(props).promise()); } + + async getOutputValue(tableName: string, key: string): Promise { + const outputResponse = await this.getItem({ + Key: { id: { S: key } }, + TableName: tableName, + AttributesToGet: ['outputValue'] + }); + if (!outputResponse.Item) { + return; + } + return outputResponse.Item.outputValue; + } } From 10336ac2b564a748177694c3c2da04abd672d3d3 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 21 Sep 2020 11:04:56 +0530 Subject: [PATCH 03/25] Prettier --- .../runtime/src/save-outputs-to-ssm/index.ts | 18 -------------- .../save-outputs-to-ssm/network-outputs.ts | 24 +++++++++++-------- .../runtime/src/save-outputs-to-ssm/utils.ts | 4 ++-- src/lib/common/src/aws/dynamodb.ts | 4 ++-- 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index b4922b3ed..07b38c391 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -34,21 +34,3 @@ export const handler = async (input: SaveOutputsToSsmInput) => { status: 'SUCCESS', }; }; - -handler({ - "acceleratorPrefix": "PBMMAccel-", - "outputsTableName": "PBMMAccel-Outputs", - "region": "eu-west-2", - "account": { - "arn": "arn:aws:organizations::538235518685:account/o-wdw2wt4bk9/233932606131", - "email": "nkoppula+non-alz-5-security@amazon.com", - "id": "233932606131", - "key": "security", - "name": "security", - "ou": "corerename", - "ouPath": "corerename" - }, - configCommitId: 'master', - configFilePath: 'raw/config.json', - configRepositoryName: 'PBMMAccel-Config-Repo' -}); diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 50aa0db04..ad562cb67 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -6,22 +6,26 @@ import { getOutput } from './utils'; /** * Outputs for network related deployments will be found in following phases - * + * */ - /** - * - * @param outputsTableName - * @param client - * @param config - * @param account - * + * + * @param outputsTableName + * @param client + * @param config + * @param account + * * @returns void */ -export async function saveNetworkOutputs(outputsTableName: string, dynamodb: DynamoDB, config: AcceleratorConfig, account: Account) { +export async function saveNetworkOutputs( + outputsTableName: string, + dynamodb: DynamoDB, + config: AcceleratorConfig, + account: Account, +) { const vpcConfigs = config.getVpcConfigs(); - for (const {accountKey, vpcConfig, ouKey} of vpcConfigs) { + for (const { accountKey, vpcConfig, ouKey } of vpcConfigs) { const outputs: StackOutput[] = await getOutput(outputsTableName, `${accountKey}-${vpcConfig.region}-1`, dynamodb); } } diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 349088fc8..0a8dcdfed 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -1,4 +1,4 @@ -import { StackOutput } from "@aws-accelerator/common-outputs/src/stack-output" +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export async function getOutput(tableName: string, key: string, dynamodb: DynamoDB): Promise { @@ -9,4 +9,4 @@ export async function getOutput(tableName: string, key: string, dynamodb: Dynamo } outputs.push(...JSON.parse(cfnOutputs.S)); return outputs; -} \ No newline at end of file +} diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index e5098b00b..155181141 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -62,10 +62,10 @@ export class DynamoDB { } async getOutputValue(tableName: string, key: string): Promise { - const outputResponse = await this.getItem({ + const outputResponse = await this.getItem({ Key: { id: { S: key } }, TableName: tableName, - AttributesToGet: ['outputValue'] + AttributesToGet: ['outputValue'], }); if (!outputResponse.Item) { return; From 417b7bdbcc0f1efd241624c3344489e4e396efab Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 21 Sep 2020 12:23:46 +0530 Subject: [PATCH 04/25] Update on save network outputs --- .../runtime/src/save-outputs-to-ssm/index.ts | 32 ++++++++++++++--- .../save-outputs-to-ssm/network-outputs.ts | 34 ++++++++++++------- .../runtime/src/save-outputs-to-ssm/utils.ts | 13 +++++++ src/lib/common/src/aws/ssm.ts | 23 ++++++++++--- 4 files changed, 81 insertions(+), 21 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 07b38c391..a6e621bd3 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -1,24 +1,39 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { saveNetworkOutputs } from './network-outputs'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; account: Account; region: string; outputsTableName: string; + assumeRoleName: string; } const dynamodb = new DynamoDB(); +const sts = new STS(); export const handler = async (input: SaveOutputsToSsmInput) => { console.log(`Adding service control policy to organization...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, configRepositoryName, configFilePath, configCommitId, outputsTableName, account } = input; + const { + configRepositoryName, + configFilePath, + configCommitId, + outputsTableName, + account, + assumeRoleName, + region, + } = input; + // Remove - if prefix ends with - + const acceleratorPrefix = input.acceleratorPrefix.endsWith('-') + ? input.acceleratorPrefix.slice(0, -1) + : input.acceleratorPrefix; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -26,9 +41,18 @@ export const handler = async (input: SaveOutputsToSsmInput) => { filePath: configFilePath, commitId: configCommitId, }); - + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); // Store Network Outputs to SSM Parameter Store - await saveNetworkOutputs(outputsTableName, dynamodb, config, account); + await saveNetworkOutputs({ + acceleratorPrefix, + config, + dynamodb, + outputsTableName, + ssm, + account, + region, + }); return { status: 'SUCCESS', diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index ad562cb67..fd758e21c 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -1,8 +1,7 @@ -import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -import { AcceleratorConfig } from '@aws-accelerator/common-config'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { getOutput } from './utils'; +import { getOutput, SaveOutputsInput } from './utils'; +import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; +import { Stack } from 'aws-sdk/clients/opsworks'; /** * Outputs for network related deployments will be found in following phases @@ -18,14 +17,25 @@ import { getOutput } from './utils'; * * @returns void */ -export async function saveNetworkOutputs( - outputsTableName: string, - dynamodb: DynamoDB, - config: AcceleratorConfig, - account: Account, -) { +export async function saveNetworkOutputs(props: SaveOutputsInput) { + const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region } = props; const vpcConfigs = config.getVpcConfigs(); - for (const { accountKey, vpcConfig, ouKey } of vpcConfigs) { - const outputs: StackOutput[] = await getOutput(outputsTableName, `${accountKey}-${vpcConfig.region}-1`, dynamodb); + const localVpcConfigs = vpcConfigs.filter(vc => vc.accountKey === account.key && vc.vpcConfig.region === region); + const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-1`, dynamodb); + let index = 1; + for (const { vpcConfig, accountKey } of localVpcConfigs) { + const vpcOutput = VpcOutputFinder.tryFindOneByAccountAndRegionAndName({ + outputs, + accountKey, + vpcName: vpcConfig.name, + }); + if (!vpcOutput) { + console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); + continue; + } + await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/name`, vpcOutput.vpcName); + await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/id`, vpcOutput.vpcId); + await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/cidr`, vpcOutput.cidrBlock); + index++; } } diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 0a8dcdfed..49bbb59bb 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -1,5 +1,18 @@ +import { AcceleratorConfig } from '@aws-accelerator/common-config'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; + +export interface SaveOutputsInput { + acceleratorPrefix: string; + outputsTableName: string; + dynamodb: DynamoDB; + config: AcceleratorConfig; + account: Account; + ssm: SSM; + region: string; +} export async function getOutput(tableName: string, key: string, dynamodb: DynamoDB): Promise { const outputs: StackOutput[] = []; diff --git a/src/lib/common/src/aws/ssm.ts b/src/lib/common/src/aws/ssm.ts index 50a97a4d0..391ec3e5c 100644 --- a/src/lib/common/src/aws/ssm.ts +++ b/src/lib/common/src/aws/ssm.ts @@ -1,18 +1,19 @@ import aws from './aws-client'; -import * as sts from 'aws-sdk/clients/ssm'; +import * as ssm from 'aws-sdk/clients/ssm'; import { throttlingBackOff } from './backoff'; export class SSM { private readonly client: aws.SSM; private readonly cache: { [roleArn: string]: aws.Credentials } = {}; - constructor(credentials?: aws.Credentials) { + constructor(credentials?: aws.Credentials, region?: string) { this.client = new aws.SSM({ credentials, + region, }); } - async getParameter(name: string): Promise { + async getParameter(name: string): Promise { return throttlingBackOff(() => this.client .getParameter({ @@ -22,8 +23,8 @@ export class SSM { ); } - async getParameterHistory(name: string): Promise { - const parameterVersions: sts.ParameterHistory[] = []; + async getParameterHistory(name: string): Promise { + const parameterVersions: ssm.ParameterHistory[] = []; let token: string | undefined; do { const response = await throttlingBackOff(() => @@ -34,4 +35,16 @@ export class SSM { } while (token); return parameterVersions; } + + async putParameter(name: string, value: string): Promise { + return throttlingBackOff(() => + this.client + .putParameter({ + Name: name, + Type: 'String', + Value: value, + }) + .promise(), + ); + } } From aec38d1a9d152bdedabd8ae2b06482e89a72315d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 21 Sep 2020 19:03:59 +0530 Subject: [PATCH 05/25] Saving VPC outputs --- .../save-outputs-to-ssm/network-outputs.ts | 205 ++++++++++++++++-- src/lib/common-outputs/src/vpc.ts | 1 + src/lib/common/src/aws/ssm.ts | 1 + 3 files changed, 190 insertions(+), 17 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index fd758e21c..fe940c9a0 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -1,7 +1,19 @@ -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getStackJsonOutput, StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { getOutput, SaveOutputsInput } from './utils'; -import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; -import { Stack } from 'aws-sdk/clients/opsworks'; +import { + SecurityGroupsOutput, + VpcOutputFinder, + VpcSecurityGroupOutput, + VpcSubnetOutput, +} from '@aws-accelerator/common-outputs/src/vpc'; +import { + ResolvedVpcConfig, + SecurityGroupConfig, + RouteTableConfig, + SubnetConfig, +} from '@aws-accelerator/common-config/'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; /** * Outputs for network related deployments will be found in following phases @@ -21,21 +33,180 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region } = props; const vpcConfigs = config.getVpcConfigs(); const localVpcConfigs = vpcConfigs.filter(vc => vc.accountKey === account.key && vc.vpcConfig.region === region); - const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-1`, dynamodb); - let index = 1; - for (const { vpcConfig, accountKey } of localVpcConfigs) { - const vpcOutput = VpcOutputFinder.tryFindOneByAccountAndRegionAndName({ - outputs, - accountKey, - vpcName: vpcConfig.name, - }); - if (!vpcOutput) { - console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); + const sharedVpcConfigs = vpcConfigs.filter( + vc => + vc.accountKey != account.key && + vc.vpcConfig.region === region && + vc.ouKey === account.ou && + (vc.vpcConfig.subnets?.find(sc => sc['share-to-ou-accounts']) || + vc.vpcConfig.subnets?.find(sc => sc['share-to-specific-accounts']?.includes(account.key))), + ); + const localOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-1`, dynamodb); + let index = 0; + for (const resolvedVpcConfig of localVpcConfigs) { + if (resolvedVpcConfig.ouKey) { + await saveVpcOutputs(++index, resolvedVpcConfig, localOutputs, ssm, acceleratorPrefix, 'vpc', account); + } else { + await saveVpcOutputs(++index, resolvedVpcConfig, localOutputs, ssm, acceleratorPrefix, 'lvpc', account); + } + } + + index = 0; + if (sharedVpcConfigs.length === 0) { + return; + } + const sharedSgOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-2`, dynamodb); + const sgOutputs: SecurityGroupsOutput[] = getStackJsonOutput(sharedSgOutputs, { + accountKey: account.key, + outputType: 'SecurityGroupsOutput', + }); + for (const resolvedVpcConfig of sharedVpcConfigs) { + const rootOutputs: StackOutput[] = await getOutput( + outputsTableName, + `${resolvedVpcConfig.accountKey}-${region}-1`, + dynamodb, + ); + const vpcSgOutputs = sgOutputs.find(sg => sg.vpcName); + await saveVpcOutputs( + ++index, + resolvedVpcConfig, + rootOutputs, + ssm, + acceleratorPrefix, + 'vpc', + account, + vpcSgOutputs?.securityGroupIds, + true, + ); + } +} + +async function saveVpcOutputs( + index: number, + resolvedVpcConfig: ResolvedVpcConfig, + outputs: StackOutput[], + ssm: SSM, + acceleratorPrefix: string, + vpcPrefix: string, + account: Account, + sgOutputs?: VpcSecurityGroupOutput[], + sharedVpc?: boolean, +) { + const { accountKey, vpcConfig } = resolvedVpcConfig; + const vpcOutput = VpcOutputFinder.tryFindOneByAccountAndRegionAndName({ + outputs, + accountKey, + vpcName: vpcConfig.name, + }); + if (!vpcOutput) { + console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); + return; + } + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); + let subnetsConfig = vpcConfig.subnets; + if (sharedVpc) { + subnetsConfig = vpcConfig.subnets?.filter( + vs => vs['share-to-ou-accounts'] || vs['share-to-specific-accounts']?.includes(account.key), + ); + } + if (subnetsConfig) { + await saveSubnets(subnetsConfig, vpcOutput.subnets, ssm, index, acceleratorPrefix, vpcPrefix, vpcConfig.name); + } + + if (vpcConfig['route-tables'] && vpcOutput.routeTables) { + await saveRouteTables(vpcConfig['route-tables'], vpcOutput.routeTables, ssm, index, acceleratorPrefix, vpcPrefix); + } + + let vpcSgOutputs: VpcSecurityGroupOutput[] = vpcOutput.securityGroups; + if (sharedVpc) { + vpcSgOutputs = sgOutputs!; + } + if (vpcConfig['security-groups'] && vpcSgOutputs) { + await saveSecurityGroups(vpcConfig['security-groups'], vpcSgOutputs, ssm, index, acceleratorPrefix, vpcPrefix); + } +} + +export async function saveSecurityGroups( + securityGroupsConfig: SecurityGroupConfig[], + securityGroupsOutputs: VpcSecurityGroupOutput[], + ssm: SSM, + vpcIndex: number, + acceleratorPrefix: string, + vpcPrefix: string, +) { + let sgIndex = 0; + for (const sgConfig of securityGroupsConfig) { + const sgOutput = securityGroupsOutputs.find(sg => sg.securityGroupName === sgConfig.name); + if (!sgOutput) { + console.warn(`Didn't find SecurityGroup "${sgConfig.name}" in output`); continue; } - await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/name`, vpcOutput.vpcName); - await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/id`, vpcOutput.vpcId); - await ssm.putParameter(`/${acceleratorPrefix}/network/vpc/${index}/cidr`, vpcOutput.cidrBlock); - index++; + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/name`, + `${sgConfig.name}_sg`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/id`, + sgOutput.securityGroupId, + ); + sgIndex++; + } +} + +export async function saveRouteTables( + routeTablesConfig: RouteTableConfig[], + routeTablesOutputs: { [name: string]: string }, + ssm: SSM, + vpcIndex: number, + acceleratorPrefix: string, + vpcPrefix: string, +) { + let rtIndex = 0; + for (const routeTableConfig of routeTablesConfig) { + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/name`, + `${routeTableConfig.name}_rt`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/id`, + routeTablesOutputs[routeTableConfig.name], + ); + rtIndex++; + } +} + +export async function saveSubnets( + subnetsConfig: SubnetConfig[], + subnetOutputs: VpcSubnetOutput[], + ssm: SSM, + vpcIndex: number, + acceleratorPrefix: string, + vpcPrefix: string, + vpcName: string, +) { + let netIndex = 0; + for (const subnetConfig of subnetsConfig || []) { + for (const subnetDef of subnetConfig.definitions.filter(sn => !sn.disabled)) { + const subnetOutput = subnetOutputs.find(vs => vs.subnetName === subnetConfig.name && vs.az === subnetDef.az); + if (!subnetOutput) { + console.warn(`Didn't find subnet "${subnetConfig.name}" in output`); + continue; + } + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetDef.az}/name`, + `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetOutput.az}/id`, + subnetOutput.subnetId, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetOutput.az}/cidr`, + subnetOutput.cidrBlock, + ); + } + netIndex++; } } diff --git a/src/lib/common-outputs/src/vpc.ts b/src/lib/common-outputs/src/vpc.ts index 3e1b1ef4f..c9e367c6d 100644 --- a/src/lib/common-outputs/src/vpc.ts +++ b/src/lib/common-outputs/src/vpc.ts @@ -53,6 +53,7 @@ export const VpcOutput = t.interface( ); export type VpcOutput = t.TypeOf; +export type VpcSubnetOutput = t.TypeOf; export const VpcOutputFinder = createStructuredOutputFinder(VpcOutput, finder => ({ tryFindOneByAccountAndRegionAndName: (props: { diff --git a/src/lib/common/src/aws/ssm.ts b/src/lib/common/src/aws/ssm.ts index 391ec3e5c..df70f48b2 100644 --- a/src/lib/common/src/aws/ssm.ts +++ b/src/lib/common/src/aws/ssm.ts @@ -43,6 +43,7 @@ export class SSM { Name: name, Type: 'String', Value: value, + Overwrite: true, }) .promise(), ); From d49466aec158dfe041e0d156d08339ed6c61f747 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 22 Sep 2020 15:34:19 +0530 Subject: [PATCH 06/25] Using DDB for previous index while storing outputs --- src/core/cdk/src/initial-setup.ts | 12 + .../save-outputs-to-ssm/network-outputs.ts | 468 ++++++++++++++---- .../runtime/src/save-outputs-to-ssm/utils.ts | 5 + 3 files changed, 380 insertions(+), 105 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 74d3170d2..9998fa79b 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -90,6 +90,18 @@ export namespace InitialSetup { encryption: dynamodb.TableEncryption.DEFAULT, }); + const outputUtilsTable = new dynamodb.Table(this, 'OutputUtils', { + tableName: createName({ + name: 'Output-Utils', + suffixLength: 0, + }), + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING, + }, + encryption: dynamodb.TableEncryption.DEFAULT, + }); + // This is the maximum time before a build times out // The role used by the build should allow this session duration const buildTimeout = cdk.Duration.hours(4); diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index fe940c9a0..5f1918385 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -1,5 +1,5 @@ import { getStackJsonOutput, StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { getOutput, SaveOutputsInput } from './utils'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput } from './utils'; import { SecurityGroupsOutput, VpcOutputFinder, @@ -14,7 +14,24 @@ import { } from '@aws-accelerator/common-config/'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { getUpdateValueInput } from '../utils/dynamodb-requests'; +interface OutputUtilVpc { + name: string; + subnets: OutputUtilGenericType[]; + securityGroups: OutputUtilGenericType[]; + routeTables: OutputUtilGenericType[]; + index: number; + type: 'vpc' | 'lvpc'; +} + +interface OutputUtilNetwork { + vpcs?: OutputUtilVpc[]; +} + +function getIndex(input: T[], searchName: string) { + console.log(typeof input); +} /** * Outputs for network related deployments will be found in following phases * @@ -31,68 +48,211 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts'; */ export async function saveNetworkOutputs(props: SaveOutputsInput) { const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region } = props; + const networkOutputUtils: OutputUtilNetwork = { + vpcs: [], + }; + const removalNetworkOutputUtils: OutputUtilNetwork = { + vpcs: [...(networkOutputUtils.vpcs || [])], + }; const vpcConfigs = config.getVpcConfigs(); - const localVpcConfigs = vpcConfigs.filter(vc => vc.accountKey === account.key && vc.vpcConfig.region === region); + const localVpcConfigs = vpcConfigs.filter( + vc => vc.accountKey === account.key && vc.vpcConfig.region === region && !vc.ouKey, + ); + const localOuVpcConfigs = vpcConfigs.filter( + vc => vc.accountKey === account.key && vc.vpcConfig.region === region && vc.ouKey, + ); const sharedVpcConfigs = vpcConfigs.filter( vc => - vc.accountKey != account.key && + vc.accountKey !== account.key && vc.vpcConfig.region === region && vc.ouKey === account.ou && (vc.vpcConfig.subnets?.find(sc => sc['share-to-ou-accounts']) || vc.vpcConfig.subnets?.find(sc => sc['share-to-specific-accounts']?.includes(account.key))), ); - const localOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-1`, dynamodb); - let index = 0; + + let localOutputs: StackOutput[] = []; + if (localVpcConfigs.length > 0 || localOuVpcConfigs.length > 0) { + localOutputs = await getOutput(outputsTableName, `${account.key}-${region}-1`, dynamodb); + } + if (!networkOutputUtils.vpcs) { + networkOutputUtils.vpcs = []; + } + const lvpcIndices = networkOutputUtils.vpcs.filter(lv => lv.type === 'lvpc').flatMap(v => v.index) || []; + let lvpcMaxIndex = lvpcIndices.length === 0 ? 0 : Math.max(...lvpcIndices); for (const resolvedVpcConfig of localVpcConfigs) { - if (resolvedVpcConfig.ouKey) { - await saveVpcOutputs(++index, resolvedVpcConfig, localOutputs, ssm, acceleratorPrefix, 'vpc', account); + let currentIndex: number; + const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + vpc => vpc.type === 'lvpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, + ); + + if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; } else { - await saveVpcOutputs(++index, resolvedVpcConfig, localOutputs, ssm, acceleratorPrefix, 'lvpc', account); + currentIndex = ++lvpcMaxIndex; + } + // getIndex(networkOutputUtils.lvpcs!, '') + const vpcResult = await saveVpcOutputs({ + index: currentIndex, + resolvedVpcConfig, + outputs: localOutputs, + ssm, + acceleratorPrefix, + vpcPrefix: 'lvpc', + account, + vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + }); + if (vpcResult) { + if (previousVpcDetailsIndex >= 0) { + networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; + } else { + networkOutputUtils.vpcs.push(vpcResult); + } } - } - index = 0; - if (sharedVpcConfigs.length === 0) { - return; + const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + vpc => vpc.type === 'lvpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, + ); + if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { + removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + } } - const sharedSgOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-2`, dynamodb); - const sgOutputs: SecurityGroupsOutput[] = getStackJsonOutput(sharedSgOutputs, { - accountKey: account.key, - outputType: 'SecurityGroupsOutput', - }); - for (const resolvedVpcConfig of sharedVpcConfigs) { - const rootOutputs: StackOutput[] = await getOutput( - outputsTableName, - `${resolvedVpcConfig.accountKey}-${region}-1`, - dynamodb, + + const vpcIndices = networkOutputUtils.vpcs.filter(vpc => vpc.type === 'vpc').flatMap(v => v.index) || []; + let vpcMaxIndex = vpcIndices.length === 0 ? 0 : Math.max(...vpcIndices); + for (const resolvedVpcConfig of localOuVpcConfigs) { + let currentIndex: number; + const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - const vpcSgOutputs = sgOutputs.find(sg => sg.vpcName); - await saveVpcOutputs( - ++index, + if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; + } else { + currentIndex = ++vpcMaxIndex; + } + const vpcResult = await saveVpcOutputs({ + index: currentIndex, resolvedVpcConfig, - rootOutputs, + outputs: localOutputs, ssm, acceleratorPrefix, - 'vpc', + vpcPrefix: 'vpc', account, - vpcSgOutputs?.securityGroupIds, - true, + vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + }); + + if (vpcResult) { + if (previousVpcDetailsIndex >= 0) { + networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; + } else { + networkOutputUtils.vpcs.push(vpcResult); + } + } + + const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); + if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { + removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + } } + if (sharedVpcConfigs.length > 0) { + const sharedSgOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-2`, dynamodb); + const sgOutputs: SecurityGroupsOutput[] = getStackJsonOutput(sharedSgOutputs, { + accountKey: account.key, + outputType: 'SecurityGroupsOutput', + }); + + for (const resolvedVpcConfig of sharedVpcConfigs) { + let currentIndex: number; + const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, + ); + if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; + } else { + currentIndex = ++vpcMaxIndex; + } + const rootOutputs: StackOutput[] = await getOutput( + outputsTableName, + `${resolvedVpcConfig.accountKey}-${region}-1`, + dynamodb, + ); + const vpcSgOutputs = sgOutputs.find(sg => sg.vpcName); + const vpcResult = await saveVpcOutputs({ + index: currentIndex, + resolvedVpcConfig, + outputs: rootOutputs, + ssm, + acceleratorPrefix, + vpcPrefix: 'vpc', + account, + sgOutputs: vpcSgOutputs?.securityGroupIds, + sharedVpc: true, + vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + }); + + if (vpcResult) { + if (previousVpcDetailsIndex >= 0) { + networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; + } else { + networkOutputUtils.vpcs.push(vpcResult); + } + } + + const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, + ); + if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { + removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + } + } + } + console.log(JSON.stringify(networkOutputUtils, null, 2)); + + const updateExpression = getUpdateValueInput([ + { + key: 'v', + name: 'outputValue', + type: 'S', + value: JSON.stringify(networkOutputUtils), + }, + ]); + await dynamodb.updateItem({ + TableName: `PBMMAccel-OutputUtils`, + Key: { + id: { S: `${account.key}-${region}-network` }, + }, + ...updateExpression, + }); } -async function saveVpcOutputs( - index: number, - resolvedVpcConfig: ResolvedVpcConfig, - outputs: StackOutput[], - ssm: SSM, - acceleratorPrefix: string, - vpcPrefix: string, - account: Account, - sgOutputs?: VpcSecurityGroupOutput[], - sharedVpc?: boolean, -) { +async function saveVpcOutputs(props: { + index: number; + resolvedVpcConfig: ResolvedVpcConfig; + outputs: StackOutput[]; + ssm: SSM; + acceleratorPrefix: string; + vpcPrefix: 'vpc' | 'lvpc'; + account: Account; + vpcUtil?: OutputUtilVpc; + sgOutputs?: VpcSecurityGroupOutput[]; + sharedVpc?: boolean; +}): Promise { + const { acceleratorPrefix, account, index, outputs, resolvedVpcConfig, ssm, vpcPrefix, sgOutputs, sharedVpc } = props; const { accountKey, vpcConfig } = resolvedVpcConfig; + let vpcUtil: OutputUtilVpc; + if (props.vpcUtil) { + vpcUtil = props.vpcUtil; + } else { + vpcUtil = { + index, + name: vpcConfig.name, + routeTables: [], + securityGroups: [], + subnets: [], + type: vpcPrefix, + }; + } const vpcOutput = VpcOutputFinder.tryFindOneByAccountAndRegionAndName({ outputs, accountKey, @@ -102,9 +262,9 @@ async function saveVpcOutputs( console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); return; } - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); + // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); + // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); + // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); let subnetsConfig = vpcConfig.subnets; if (sharedVpc) { subnetsConfig = vpcConfig.subnets?.filter( @@ -112,11 +272,28 @@ async function saveVpcOutputs( ); } if (subnetsConfig) { - await saveSubnets(subnetsConfig, vpcOutput.subnets, ssm, index, acceleratorPrefix, vpcPrefix, vpcConfig.name); + vpcUtil.subnets = await saveSubnets({ + subnetsConfig, + subnetOutputs: vpcOutput.subnets, + ssm, + vpcIndex: index, + acceleratorPrefix, + vpcPrefix, + vpcName: vpcConfig.name, + subnetsUtil: vpcUtil.subnets, + }); } if (vpcConfig['route-tables'] && vpcOutput.routeTables) { - await saveRouteTables(vpcConfig['route-tables'], vpcOutput.routeTables, ssm, index, acceleratorPrefix, vpcPrefix); + vpcUtil.routeTables = await saveRouteTables({ + routeTablesConfig: vpcConfig['route-tables'], + routeTablesOutputs: vpcOutput.routeTables, + ssm, + vpcIndex: index, + acceleratorPrefix, + vpcPrefix, + routeTablesUtil: vpcUtil.routeTables, + }); } let vpcSgOutputs: VpcSecurityGroupOutput[] = vpcOutput.securityGroups; @@ -124,89 +301,170 @@ async function saveVpcOutputs( vpcSgOutputs = sgOutputs!; } if (vpcConfig['security-groups'] && vpcSgOutputs) { - await saveSecurityGroups(vpcConfig['security-groups'], vpcSgOutputs, ssm, index, acceleratorPrefix, vpcPrefix); + vpcUtil.securityGroups = await saveSecurityGroups({ + securityGroupsConfig: vpcConfig['security-groups'], + securityGroupsOutputs: vpcSgOutputs, + ssm, + vpcIndex: index, + acceleratorPrefix, + vpcPrefix, + securityGroupsUtil: vpcUtil.securityGroups, + }); } + return vpcUtil; } -export async function saveSecurityGroups( - securityGroupsConfig: SecurityGroupConfig[], - securityGroupsOutputs: VpcSecurityGroupOutput[], - ssm: SSM, - vpcIndex: number, - acceleratorPrefix: string, - vpcPrefix: string, -) { - let sgIndex = 0; +export async function saveSecurityGroups(props: { + securityGroupsConfig: SecurityGroupConfig[]; + securityGroupsOutputs: VpcSecurityGroupOutput[]; + ssm: SSM; + vpcIndex: number; + acceleratorPrefix: string; + vpcPrefix: string; + securityGroupsUtil: OutputUtilGenericType[]; +}): Promise { + const { + acceleratorPrefix, + securityGroupsConfig, + securityGroupsOutputs, + ssm, + vpcIndex, + vpcPrefix, + securityGroupsUtil, + } = props; + const sgIndices = securityGroupsUtil.flatMap(r => r.index) || []; + let sgMaxIndex = sgIndices.length === 0 ? 0 : Math.max(...sgIndices); + const removalSgUtils = [...securityGroupsUtil]; + const updatedObjects: OutputUtilGenericType[] = []; for (const sgConfig of securityGroupsConfig) { + let currentIndex: number; + const previousSgDetailsIndex = securityGroupsUtil.findIndex(sg => sg.name === sgConfig.name); + if (previousSgDetailsIndex && previousSgDetailsIndex >= 0) { + currentIndex = securityGroupsUtil[previousSgDetailsIndex].index; + } else { + currentIndex = ++sgMaxIndex; + } + updatedObjects.push({ + index: currentIndex, + name: sgConfig.name, + }); const sgOutput = securityGroupsOutputs.find(sg => sg.securityGroupName === sgConfig.name); if (!sgOutput) { console.warn(`Didn't find SecurityGroup "${sgConfig.name}" in output`); continue; } - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/name`, - `${sgConfig.name}_sg`, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/id`, - sgOutput.securityGroupId, - ); - sgIndex++; + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/name`, + // `${sgConfig.name}_sg`, + // ); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/id`, + // sgOutput.securityGroupId, + // ); + const removalSgDetailsIndex = removalSgUtils?.findIndex(r => r.name === sgConfig.name); + if (removalSgDetailsIndex && removalSgDetailsIndex >= 0) { + removalSgUtils?.splice(removalSgDetailsIndex); + } } + return updatedObjects; } -export async function saveRouteTables( - routeTablesConfig: RouteTableConfig[], - routeTablesOutputs: { [name: string]: string }, - ssm: SSM, - vpcIndex: number, - acceleratorPrefix: string, - vpcPrefix: string, -) { - let rtIndex = 0; +export async function saveRouteTables(props: { + routeTablesConfig: RouteTableConfig[]; + routeTablesOutputs: { [name: string]: string }; + ssm: SSM; + vpcIndex: number; + acceleratorPrefix: string; + vpcPrefix: string; + routeTablesUtil: OutputUtilGenericType[]; +}): Promise { + const { acceleratorPrefix, routeTablesConfig, routeTablesOutputs, routeTablesUtil, ssm, vpcIndex, vpcPrefix } = props; + const routeTableIndices = routeTablesUtil.flatMap(r => r.index) || []; + let rtMaxIndex = routeTableIndices.length === 0 ? 0 : Math.max(...routeTableIndices); + const removalObjects = [...routeTablesUtil]; + const updatedObjects: OutputUtilGenericType[] = []; for (const routeTableConfig of routeTablesConfig) { - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/name`, - `${routeTableConfig.name}_rt`, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/id`, - routeTablesOutputs[routeTableConfig.name], - ); - rtIndex++; + let currentIndex: number; + const previousRtDetailsIndex = routeTablesUtil.findIndex(r => r.name === routeTableConfig.name); + if (previousRtDetailsIndex && previousRtDetailsIndex >= 0) { + currentIndex = routeTablesUtil[previousRtDetailsIndex].index; + } else { + currentIndex = ++rtMaxIndex; + } + updatedObjects.push({ + index: currentIndex, + name: routeTableConfig.name, + }); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/name`, + // `${routeTableConfig.name}_rt`, + // ); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/id`, + // routeTablesOutputs[routeTableConfig.name], + // ); + const removalRtDetailsIndex = removalObjects?.findIndex(r => r.name === routeTableConfig.name); + if (removalRtDetailsIndex && removalRtDetailsIndex >= 0) { + removalObjects?.splice(removalRtDetailsIndex); + } } + + for (const removeObject of removalObjects) { + // TODO: Remove from SSM + } + return updatedObjects; } -export async function saveSubnets( - subnetsConfig: SubnetConfig[], - subnetOutputs: VpcSubnetOutput[], - ssm: SSM, - vpcIndex: number, - acceleratorPrefix: string, - vpcPrefix: string, - vpcName: string, -) { - let netIndex = 0; +export async function saveSubnets(props: { + subnetsConfig: SubnetConfig[]; + subnetOutputs: VpcSubnetOutput[]; + ssm: SSM; + vpcIndex: number; + acceleratorPrefix: string; + vpcPrefix: string; + vpcName: string; + subnetsUtil: OutputUtilGenericType[]; +}): Promise { + const { acceleratorPrefix, ssm, subnetOutputs, subnetsConfig, vpcIndex, vpcName, vpcPrefix, subnetsUtil } = props; + const subnetIndices = subnetsUtil.flatMap(s => s.index) || []; + let subnetMaxIndex = subnetIndices.length === 0 ? 0 : Math.max(...subnetIndices); + const removalSubnetUtils = [...subnetsUtil]; + const updatedObjects: OutputUtilGenericType[] = []; for (const subnetConfig of subnetsConfig || []) { + let currentIndex: number; + const previousSubnetDetailsIndex = subnetsUtil.findIndex(s => s.name === subnetConfig.name); + if (previousSubnetDetailsIndex && previousSubnetDetailsIndex >= 0) { + currentIndex = subnetsUtil[previousSubnetDetailsIndex].index; + } else { + currentIndex = ++subnetMaxIndex; + } + updatedObjects.push({ + index: currentIndex, + name: subnetConfig.name, + }); for (const subnetDef of subnetConfig.definitions.filter(sn => !sn.disabled)) { const subnetOutput = subnetOutputs.find(vs => vs.subnetName === subnetConfig.name && vs.az === subnetDef.az); if (!subnetOutput) { console.warn(`Didn't find subnet "${subnetConfig.name}" in output`); continue; } - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetDef.az}/name`, - `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetOutput.az}/id`, - subnetOutput.subnetId, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${netIndex + 1}/az${subnetOutput.az}/cidr`, - subnetOutput.cidrBlock, - ); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetDef.az}/name`, + // `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, + // ); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/id`, + // subnetOutput.subnetId, + // ); + // await ssm.putParameter( + // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/cidr`, + // subnetOutput.cidrBlock, + // ); + } + const removalSubnetDetailsIndex = subnetsUtil?.findIndex(s => s.name === subnetConfig.name); + if (removalSubnetDetailsIndex && removalSubnetDetailsIndex >= 0) { + removalSubnetUtils?.splice(removalSubnetDetailsIndex); } - netIndex++; } + return updatedObjects; } diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 49bbb59bb..1343a10cc 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -14,6 +14,11 @@ export interface SaveOutputsInput { region: string; } +export interface OutputUtilGenericType { + name: string; + index: number; +} + export async function getOutput(tableName: string, key: string, dynamodb: DynamoDB): Promise { const outputs: StackOutput[] = []; const cfnOutputs = await dynamodb.getOutputValue(tableName, key); From 4e098164a9f56a3c1be971c8c49eba77659c7b03 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 22 Sep 2020 16:45:02 +0530 Subject: [PATCH 07/25] updates --- .../runtime/src/save-outputs-to-ssm/index.ts | 3 + .../save-outputs-to-ssm/network-outputs.ts | 135 +++++++++--------- .../runtime/src/save-outputs-to-ssm/utils.ts | 9 ++ 3 files changed, 82 insertions(+), 65 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index a6e621bd3..c36db3a30 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -12,6 +12,7 @@ export interface SaveOutputsToSsmInput extends LoadConfigurationInput { region: string; outputsTableName: string; assumeRoleName: string; + outputUtilsTableName: string; } const dynamodb = new DynamoDB(); @@ -29,6 +30,7 @@ export const handler = async (input: SaveOutputsToSsmInput) => { account, assumeRoleName, region, + outputUtilsTableName, } = input; // Remove - if prefix ends with - const acceleratorPrefix = input.acceleratorPrefix.endsWith('-') @@ -52,6 +54,7 @@ export const handler = async (input: SaveOutputsToSsmInput) => { ssm, account, region, + outputUtilsTableName, }); return { diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 5f1918385..00dc34a69 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -1,5 +1,5 @@ import { getStackJsonOutput, StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { getOutput, OutputUtilGenericType, SaveOutputsInput } from './utils'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, getOutputUtil } from './utils'; import { SecurityGroupsOutput, VpcOutputFinder, @@ -47,13 +47,31 @@ function getIndex(input: T[], searchName: string) { * @returns void */ export async function saveNetworkOutputs(props: SaveOutputsInput) { - const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region } = props; - const networkOutputUtils: OutputUtilNetwork = { + const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region, outputUtilsTableName } = props; + const oldNetworkOutputUtils = await getOutputUtil(outputUtilsTableName, `${account.key}-${region}-network`, dynamodb); + // Existing index check happens on this variable + let networkOutputUtils: OutputUtilNetwork; + if (oldNetworkOutputUtils) { + networkOutputUtils = oldNetworkOutputUtils; + } else { + networkOutputUtils = { + vpcs: [], + }; + } + + // Storing new resource index and updating DDB in this variable + const newNetworkOutputs: OutputUtilNetwork = { vpcs: [], }; - const removalNetworkOutputUtils: OutputUtilNetwork = { + if (!newNetworkOutputs.vpcs) { + newNetworkOutputs.vpcs = []; + } + + // Removal from SSM Parameter store happens on left over in this variable + const removalObjects: OutputUtilNetwork = { vpcs: [...(networkOutputUtils.vpcs || [])], }; + const vpcConfigs = config.getVpcConfigs(); const localVpcConfigs = vpcConfigs.filter( vc => vc.accountKey === account.key && vc.vpcConfig.region === region && !vc.ouKey, @@ -81,16 +99,15 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { let lvpcMaxIndex = lvpcIndices.length === 0 ? 0 : Math.max(...lvpcIndices); for (const resolvedVpcConfig of localVpcConfigs) { let currentIndex: number; - const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + const previousIndex = networkOutputUtils.vpcs.findIndex( vpc => vpc.type === 'lvpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - - if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { - currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; + if (previousIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousIndex].index; } else { currentIndex = ++lvpcMaxIndex; } - // getIndex(networkOutputUtils.lvpcs!, '') + const vpcResult = await saveVpcOutputs({ index: currentIndex, resolvedVpcConfig, @@ -99,21 +116,17 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { acceleratorPrefix, vpcPrefix: 'lvpc', account, - vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { - if (previousVpcDetailsIndex >= 0) { - networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; - } else { - networkOutputUtils.vpcs.push(vpcResult); - } + newNetworkOutputs.vpcs.push(vpcResult); } - const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + const removalIndex = removalObjects.vpcs?.findIndex( vpc => vpc.type === 'lvpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { - removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + if (removalIndex! >= 0) { + removalObjects.vpcs?.splice(removalIndex!); } } @@ -121,11 +134,11 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { let vpcMaxIndex = vpcIndices.length === 0 ? 0 : Math.max(...vpcIndices); for (const resolvedVpcConfig of localOuVpcConfigs) { let currentIndex: number; - const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + const previousIndex = networkOutputUtils.vpcs.findIndex( vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { - currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; + if (previousIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousIndex].index; } else { currentIndex = ++vpcMaxIndex; } @@ -137,22 +150,18 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { acceleratorPrefix, vpcPrefix: 'vpc', account, - vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { - if (previousVpcDetailsIndex >= 0) { - networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; - } else { - networkOutputUtils.vpcs.push(vpcResult); - } + newNetworkOutputs.vpcs.push(vpcResult); } - const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + const removalIndex = removalObjects.vpcs?.findIndex( vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { - removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + if (removalIndex! >= 0) { + removalObjects.vpcs?.splice(removalIndex!); } } if (sharedVpcConfigs.length > 0) { @@ -164,11 +173,11 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { for (const resolvedVpcConfig of sharedVpcConfigs) { let currentIndex: number; - const previousVpcDetailsIndex = networkOutputUtils.vpcs.findIndex( + const previousIndex = networkOutputUtils.vpcs.findIndex( vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - if (previousVpcDetailsIndex && previousVpcDetailsIndex >= 0) { - currentIndex = networkOutputUtils.vpcs[previousVpcDetailsIndex].index; + if (previousIndex >= 0) { + currentIndex = networkOutputUtils.vpcs[previousIndex].index; } else { currentIndex = ++vpcMaxIndex; } @@ -188,37 +197,33 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { account, sgOutputs: vpcSgOutputs?.securityGroupIds, sharedVpc: true, - vpcUtil: previousVpcDetailsIndex ? networkOutputUtils.vpcs[previousVpcDetailsIndex] : undefined, + vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { - if (previousVpcDetailsIndex >= 0) { - networkOutputUtils.vpcs[previousVpcDetailsIndex] = vpcResult; - } else { - networkOutputUtils.vpcs.push(vpcResult); - } + newNetworkOutputs.vpcs.push(vpcResult); } - const removalVpcDetailsIndex = removalNetworkOutputUtils.vpcs?.findIndex( + const removalIndex = removalObjects.vpcs?.findIndex( vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); - if (removalVpcDetailsIndex && removalVpcDetailsIndex >= 0) { - removalNetworkOutputUtils.vpcs?.splice(removalVpcDetailsIndex); + if (removalIndex! >= 0) { + removalObjects.vpcs?.splice(removalIndex!); } } } - console.log(JSON.stringify(networkOutputUtils, null, 2)); + // console.log(JSON.stringify(newNetworkOutputs, null, 2)); const updateExpression = getUpdateValueInput([ { key: 'v', name: 'outputValue', type: 'S', - value: JSON.stringify(networkOutputUtils), + value: JSON.stringify(newNetworkOutputs), }, ]); await dynamodb.updateItem({ - TableName: `PBMMAccel-OutputUtils`, + TableName: outputUtilsTableName, Key: { id: { S: `${account.key}-${region}-network` }, }, @@ -334,13 +339,13 @@ export async function saveSecurityGroups(props: { } = props; const sgIndices = securityGroupsUtil.flatMap(r => r.index) || []; let sgMaxIndex = sgIndices.length === 0 ? 0 : Math.max(...sgIndices); - const removalSgUtils = [...securityGroupsUtil]; + const removalObjects = [...securityGroupsUtil]; const updatedObjects: OutputUtilGenericType[] = []; for (const sgConfig of securityGroupsConfig) { let currentIndex: number; - const previousSgDetailsIndex = securityGroupsUtil.findIndex(sg => sg.name === sgConfig.name); - if (previousSgDetailsIndex && previousSgDetailsIndex >= 0) { - currentIndex = securityGroupsUtil[previousSgDetailsIndex].index; + const previousIndex = securityGroupsUtil.findIndex(sg => sg.name === sgConfig.name); + if (previousIndex >= 0) { + currentIndex = securityGroupsUtil[previousIndex].index; } else { currentIndex = ++sgMaxIndex; } @@ -361,9 +366,9 @@ export async function saveSecurityGroups(props: { // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/id`, // sgOutput.securityGroupId, // ); - const removalSgDetailsIndex = removalSgUtils?.findIndex(r => r.name === sgConfig.name); - if (removalSgDetailsIndex && removalSgDetailsIndex >= 0) { - removalSgUtils?.splice(removalSgDetailsIndex); + const removalIndex = removalObjects?.findIndex(r => r.name === sgConfig.name); + if (removalIndex >= 0) { + removalObjects?.splice(removalIndex); } } return updatedObjects; @@ -385,9 +390,9 @@ export async function saveRouteTables(props: { const updatedObjects: OutputUtilGenericType[] = []; for (const routeTableConfig of routeTablesConfig) { let currentIndex: number; - const previousRtDetailsIndex = routeTablesUtil.findIndex(r => r.name === routeTableConfig.name); - if (previousRtDetailsIndex && previousRtDetailsIndex >= 0) { - currentIndex = routeTablesUtil[previousRtDetailsIndex].index; + const previousIndex = routeTablesUtil.findIndex(r => r.name === routeTableConfig.name); + if (previousIndex >= 0) { + currentIndex = routeTablesUtil[previousIndex].index; } else { currentIndex = ++rtMaxIndex; } @@ -403,9 +408,9 @@ export async function saveRouteTables(props: { // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/id`, // routeTablesOutputs[routeTableConfig.name], // ); - const removalRtDetailsIndex = removalObjects?.findIndex(r => r.name === routeTableConfig.name); - if (removalRtDetailsIndex && removalRtDetailsIndex >= 0) { - removalObjects?.splice(removalRtDetailsIndex); + const removalIndex = removalObjects?.findIndex(r => r.name === routeTableConfig.name); + if (removalIndex >= 0) { + removalObjects?.splice(removalIndex); } } @@ -428,13 +433,13 @@ export async function saveSubnets(props: { const { acceleratorPrefix, ssm, subnetOutputs, subnetsConfig, vpcIndex, vpcName, vpcPrefix, subnetsUtil } = props; const subnetIndices = subnetsUtil.flatMap(s => s.index) || []; let subnetMaxIndex = subnetIndices.length === 0 ? 0 : Math.max(...subnetIndices); - const removalSubnetUtils = [...subnetsUtil]; + const removalObjects = [...subnetsUtil]; const updatedObjects: OutputUtilGenericType[] = []; for (const subnetConfig of subnetsConfig || []) { let currentIndex: number; - const previousSubnetDetailsIndex = subnetsUtil.findIndex(s => s.name === subnetConfig.name); - if (previousSubnetDetailsIndex && previousSubnetDetailsIndex >= 0) { - currentIndex = subnetsUtil[previousSubnetDetailsIndex].index; + const previousIndex = subnetsUtil.findIndex(s => s.name === subnetConfig.name); + if (previousIndex >= 0) { + currentIndex = subnetsUtil[previousIndex].index; } else { currentIndex = ++subnetMaxIndex; } @@ -461,9 +466,9 @@ export async function saveSubnets(props: { // subnetOutput.cidrBlock, // ); } - const removalSubnetDetailsIndex = subnetsUtil?.findIndex(s => s.name === subnetConfig.name); - if (removalSubnetDetailsIndex && removalSubnetDetailsIndex >= 0) { - removalSubnetUtils?.splice(removalSubnetDetailsIndex); + const removalIndex = removalObjects?.findIndex(s => s.name === subnetConfig.name); + if (removalIndex >= 0) { + removalObjects?.splice(removalIndex); } } return updatedObjects; diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 1343a10cc..c97dd4e77 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -12,6 +12,7 @@ export interface SaveOutputsInput { account: Account; ssm: SSM; region: string; + outputUtilsTableName: string; } export interface OutputUtilGenericType { @@ -28,3 +29,11 @@ export async function getOutput(tableName: string, key: string, dynamodb: Dynamo outputs.push(...JSON.parse(cfnOutputs.S)); return outputs; } + +export async function getOutputUtil(tableName: string, key: string, dynamodb: DynamoDB) { + const outputUtils = await dynamodb.getOutputValue(tableName, key); + if (!outputUtils || !outputUtils.S) { + return; + } + return JSON.parse(outputUtils.S); +} From 7b50b8c3264965171ccbfb700e9334447e824513 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 22 Sep 2020 19:01:37 +0530 Subject: [PATCH 08/25] adding remove prevous ssm and also code for routeTables --- .../save-outputs-to-ssm/network-outputs.ts | 164 +++++++----------- src/lib/common/src/aws/ssm.ts | 10 ++ 2 files changed, 75 insertions(+), 99 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 00dc34a69..36555171c 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -6,21 +6,18 @@ import { VpcSecurityGroupOutput, VpcSubnetOutput, } from '@aws-accelerator/common-outputs/src/vpc'; -import { - ResolvedVpcConfig, - SecurityGroupConfig, - RouteTableConfig, - SubnetConfig, -} from '@aws-accelerator/common-config/'; +import { ResolvedVpcConfig, SecurityGroupConfig, SubnetConfig } from '@aws-accelerator/common-config/'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { getUpdateValueInput } from '../utils/dynamodb-requests'; +interface OutputUtilSubnet extends OutputUtilGenericType { + azs: string[]; +} interface OutputUtilVpc { name: string; - subnets: OutputUtilGenericType[]; + subnets: OutputUtilSubnet[]; securityGroups: OutputUtilGenericType[]; - routeTables: OutputUtilGenericType[]; index: number; type: 'vpc' | 'lvpc'; } @@ -125,8 +122,9 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { const removalIndex = removalObjects.vpcs?.findIndex( vpc => vpc.type === 'lvpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); + if (removalIndex! >= 0) { - removalObjects.vpcs?.splice(removalIndex!); + removalObjects.vpcs?.splice(removalIndex!, 1); } } @@ -161,7 +159,7 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); if (removalIndex! >= 0) { - removalObjects.vpcs?.splice(removalIndex!); + removalObjects.vpcs?.splice(removalIndex!, 1); } } if (sharedVpcConfigs.length > 0) { @@ -208,11 +206,10 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { vpc => vpc.type === 'vpc' && vpc.name === resolvedVpcConfig.vpcConfig.name, ); if (removalIndex! >= 0) { - removalObjects.vpcs?.splice(removalIndex!); + removalObjects.vpcs?.splice(removalIndex!, 1); } } } - // console.log(JSON.stringify(newNetworkOutputs, null, 2)); const updateExpression = getUpdateValueInput([ { @@ -229,6 +226,33 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { }, ...updateExpression, }); + for (const removeObject of removalObjects.vpcs || []) { + const removalSgs = removeObject.securityGroups + .map(sg => [ + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/sg/${sg.index}/name`, + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/sg/${sg.index}/id`, + ]) + .flatMap(s => s); + const removalSns = removeObject.subnets + .map(sn => + sn.azs.map(snz => [ + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/net/${sn.index}/az${snz}/name`, + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/net/${sn.index}/az${snz}/id`, + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/net/${sn.index}/az${snz}/cidr`, + ]), + ) + .flatMap(azSn => azSn) + .flatMap(sn => sn); + const removalVpc = [ + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/name`, + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/id`, + `/${acceleratorPrefix}/network/${removeObject.type}/${removeObject.index}/cidr`, + ]; + const removeNames = [...removalSgs, ...removalSns, ...removalVpc]; + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } + } } async function saveVpcOutputs(props: { @@ -252,7 +276,6 @@ async function saveVpcOutputs(props: { vpcUtil = { index, name: vpcConfig.name, - routeTables: [], securityGroups: [], subnets: [], type: vpcPrefix, @@ -267,9 +290,9 @@ async function saveVpcOutputs(props: { console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); return; } - // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); - // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); - // await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); let subnetsConfig = vpcConfig.subnets; if (sharedVpc) { subnetsConfig = vpcConfig.subnets?.filter( @@ -289,18 +312,6 @@ async function saveVpcOutputs(props: { }); } - if (vpcConfig['route-tables'] && vpcOutput.routeTables) { - vpcUtil.routeTables = await saveRouteTables({ - routeTablesConfig: vpcConfig['route-tables'], - routeTablesOutputs: vpcOutput.routeTables, - ssm, - vpcIndex: index, - acceleratorPrefix, - vpcPrefix, - routeTablesUtil: vpcUtil.routeTables, - }); - } - let vpcSgOutputs: VpcSecurityGroupOutput[] = vpcOutput.securityGroups; if (sharedVpc) { vpcSgOutputs = sgOutputs!; @@ -358,65 +369,19 @@ export async function saveSecurityGroups(props: { console.warn(`Didn't find SecurityGroup "${sgConfig.name}" in output`); continue; } - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/name`, - // `${sgConfig.name}_sg`, - // ); - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sgIndex + 1}/id`, - // sgOutput.securityGroupId, - // ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/name`, + `${sgConfig.name}_sg`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/id`, + sgOutput.securityGroupId, + ); const removalIndex = removalObjects?.findIndex(r => r.name === sgConfig.name); if (removalIndex >= 0) { - removalObjects?.splice(removalIndex); - } - } - return updatedObjects; -} - -export async function saveRouteTables(props: { - routeTablesConfig: RouteTableConfig[]; - routeTablesOutputs: { [name: string]: string }; - ssm: SSM; - vpcIndex: number; - acceleratorPrefix: string; - vpcPrefix: string; - routeTablesUtil: OutputUtilGenericType[]; -}): Promise { - const { acceleratorPrefix, routeTablesConfig, routeTablesOutputs, routeTablesUtil, ssm, vpcIndex, vpcPrefix } = props; - const routeTableIndices = routeTablesUtil.flatMap(r => r.index) || []; - let rtMaxIndex = routeTableIndices.length === 0 ? 0 : Math.max(...routeTableIndices); - const removalObjects = [...routeTablesUtil]; - const updatedObjects: OutputUtilGenericType[] = []; - for (const routeTableConfig of routeTablesConfig) { - let currentIndex: number; - const previousIndex = routeTablesUtil.findIndex(r => r.name === routeTableConfig.name); - if (previousIndex >= 0) { - currentIndex = routeTablesUtil[previousIndex].index; - } else { - currentIndex = ++rtMaxIndex; - } - updatedObjects.push({ - index: currentIndex, - name: routeTableConfig.name, - }); - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/name`, - // `${routeTableConfig.name}_rt`, - // ); - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/rt/${rtIndex + 1}/id`, - // routeTablesOutputs[routeTableConfig.name], - // ); - const removalIndex = removalObjects?.findIndex(r => r.name === routeTableConfig.name); - if (removalIndex >= 0) { - removalObjects?.splice(removalIndex); + removalObjects?.splice(removalIndex, 1); } } - - for (const removeObject of removalObjects) { - // TODO: Remove from SSM - } return updatedObjects; } @@ -428,13 +393,13 @@ export async function saveSubnets(props: { acceleratorPrefix: string; vpcPrefix: string; vpcName: string; - subnetsUtil: OutputUtilGenericType[]; -}): Promise { + subnetsUtil: OutputUtilSubnet[]; +}): Promise { const { acceleratorPrefix, ssm, subnetOutputs, subnetsConfig, vpcIndex, vpcName, vpcPrefix, subnetsUtil } = props; const subnetIndices = subnetsUtil.flatMap(s => s.index) || []; let subnetMaxIndex = subnetIndices.length === 0 ? 0 : Math.max(...subnetIndices); const removalObjects = [...subnetsUtil]; - const updatedObjects: OutputUtilGenericType[] = []; + const updatedObjects: OutputUtilSubnet[] = []; for (const subnetConfig of subnetsConfig || []) { let currentIndex: number; const previousIndex = subnetsUtil.findIndex(s => s.name === subnetConfig.name); @@ -446,6 +411,7 @@ export async function saveSubnets(props: { updatedObjects.push({ index: currentIndex, name: subnetConfig.name, + azs: subnetConfig.definitions.filter(sn => !sn.disabled).map(s => s.az), }); for (const subnetDef of subnetConfig.definitions.filter(sn => !sn.disabled)) { const subnetOutput = subnetOutputs.find(vs => vs.subnetName === subnetConfig.name && vs.az === subnetDef.az); @@ -453,22 +419,22 @@ export async function saveSubnets(props: { console.warn(`Didn't find subnet "${subnetConfig.name}" in output`); continue; } - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetDef.az}/name`, - // `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, - // ); - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/id`, - // subnetOutput.subnetId, - // ); - // await ssm.putParameter( - // `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/cidr`, - // subnetOutput.cidrBlock, - // ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetDef.az}/name`, + `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/id`, + subnetOutput.subnetId, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/cidr`, + subnetOutput.cidrBlock, + ); } const removalIndex = removalObjects?.findIndex(s => s.name === subnetConfig.name); if (removalIndex >= 0) { - removalObjects?.splice(removalIndex); + removalObjects?.splice(removalIndex, 1); } } return updatedObjects; diff --git a/src/lib/common/src/aws/ssm.ts b/src/lib/common/src/aws/ssm.ts index df70f48b2..7547e95c0 100644 --- a/src/lib/common/src/aws/ssm.ts +++ b/src/lib/common/src/aws/ssm.ts @@ -48,4 +48,14 @@ export class SSM { .promise(), ); } + + async deleteParameters(names: string[]): Promise { + await throttlingBackOff(() => + this.client + .deleteParameters({ + Names: names, + }) + .promise(), + ); + } } From 8020fc086ff5a173e8ee5f9d9afa5d5a9b91c1f9 Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 22 Sep 2020 21:10:28 +0530 Subject: [PATCH 09/25] added ssm params for iam --- reference-artifacts/config.example.json | 1 + .../src/save-outputs-to-ssm/iam-outputs.ts | 118 ++++++++ .../src/save-outputs-to-ssm/iam-utils.ts | 270 ++++++++++++++++++ .../runtime/src/save-outputs-to-ssm/index.ts | 42 ++- .../runtime/src/save-outputs-to-ssm/utils.ts | 21 +- src/deployments/cdk/src/common/iam-assets.ts | 29 ++ .../cdk/src/deployments/iam/outputs.ts | 9 +- src/lib/common-config/src/index.ts | 1 + src/lib/common-outputs/src/iam-role.ts | 29 ++ src/lib/common-outputs/src/iam-users.ts | 43 +++ 10 files changed, 551 insertions(+), 12 deletions(-) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts create mode 100644 src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts create mode 100644 src/lib/common-outputs/src/iam-users.ts diff --git a/reference-artifacts/config.example.json b/reference-artifacts/config.example.json index c89c3ba26..3640bbc98 100644 --- a/reference-artifacts/config.example.json +++ b/reference-artifacts/config.example.json @@ -11,6 +11,7 @@ "workloadaccounts-prefix": "config", "workloadaccounts-param-filename": "config.json", "ignored-ous": [], + "additional-global-output-regions": [], "supported-regions": [ "ap-northeast-1", "ap-northeast-2", diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts new file mode 100644 index 000000000..b0aa0bbc6 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts @@ -0,0 +1,118 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput, SaveOutputsInput, saveIndexOutput, OutputUtilGenericType, getIamSsmOutput } from './utils'; +import { IamConfig } from '@aws-accelerator/common-config'; +import { + prepareMandatoryAccountIamConfigs, + prepareOuAccountIamConfigs, + saveIamRoles, + saveIamPolicy, + saveIamUsers, + saveIamGroups, +} from './iam-utils'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; + +interface IamOutput { + roles: OutputUtilGenericType[]; + policies: OutputUtilGenericType[]; + users: OutputUtilGenericType[]; + groups: OutputUtilGenericType[]; +} + +/** + * Outputs for IAM related deployments will be found in following phases + * Phase 1 + */ + +/** + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveIamOutputs(props: SaveOutputsInput) { + const { acceleratorPrefix, account, config, dynamodb, outputsTableName, assumeRoleName, region, outputUtilsTableName } = props; + + const accountConfig = config.getMandatoryAccountConfigs().find(([accountKey, _]) => accountKey === account.key); + const accountsIam: { [accountKey: string]: IamConfig[] } = {}; + + // finding iam for account iam configs + prepareMandatoryAccountIamConfigs(accountsIam, accountConfig); + + // finding iam for ou iam configs + prepareOuAccountIamConfigs(config, account, accountsIam); + // console.log('Accounts Iam', Object.keys(accountsIam)); + + // if no IAM Config found for the account, return + if (Object.keys(accountsIam).length === 0) { + return; + } + + const smRegion = config["global-options"]["aws-org-master"].region; + const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${smRegion}-1`, dynamodb); + const ssmOutputs = await getIamSsmOutput(outputUtilsTableName, `${account.key}-${region}-identity`, dynamodb); + // console.log('ssmOutputs', ssmOutputs); + + const roles: OutputUtilGenericType[] = []; + const policies: OutputUtilGenericType[] = []; + const users: OutputUtilGenericType[] = []; + const groups: OutputUtilGenericType[] = []; + + if (ssmOutputs) { + const ssmIamOutput: IamOutput = JSON.parse(ssmOutputs); + roles.push(...ssmIamOutput.roles); + policies.push(...ssmIamOutput.policies); + users.push(...ssmIamOutput.users); + groups.push(...ssmIamOutput.groups); + } + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + + const updatedRoles = await saveIamRoles( + Object.values(accountsIam)[0], + outputs, + ssm, + account.key, + acceleratorPrefix, + roles, + ); + const updatedPolicies = await saveIamPolicy( + Object.values(accountsIam)[0], + outputs, + ssm, + account.key, + acceleratorPrefix, + policies, + ); + const updatedUsers = await saveIamUsers( + Object.values(accountsIam)[0], + outputs, + ssm, + account.key, + acceleratorPrefix, + users, + ); + const updatedGroups = await saveIamGroups( + Object.values(accountsIam)[0], + outputs, + ssm, + account.key, + acceleratorPrefix, + groups, + ); + + const iamOutput: IamOutput = { + roles: updatedRoles, + policies: updatedPolicies, + users: updatedUsers, + groups: updatedGroups, + }; + const iamIndexOutput = JSON.stringify(iamOutput); + console.log('indexOutput', iamIndexOutput); + saveIndexOutput(outputUtilsTableName, `${account.key}-${region}-identity`, iamIndexOutput, dynamodb); +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts new file mode 100644 index 000000000..680873c14 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -0,0 +1,270 @@ +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { IamConfig, AccountConfig, AcceleratorConfig } from '@aws-accelerator/common-config'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { IamGroupOutputFinder, IamUserOutputFinder } from '@aws-accelerator/common-outputs/src/iam-users'; +import { IamPolicyOutputFinder, IamRoleNameOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; +import { OutputUtilGenericType } from './utils'; + +export async function prepareMandatoryAccountIamConfigs( + accountsIamConfig: { [accountKey: string]: IamConfig[] }, + accountConfig: [string, AccountConfig] | undefined, +) { + if (!accountConfig) { + return; + } + const iamConfig = accountConfig[1].iam; + if (!iamConfig) { + return; + } + if (!accountsIamConfig[accountConfig[0]]) { + accountsIamConfig[accountConfig[0]] = [iamConfig]; + } else { + accountsIamConfig[accountConfig[0]].push(iamConfig); + } +} + +export async function prepareOuAccountIamConfigs( + config: AcceleratorConfig, + account: Account, + accountsIamConfig: { [accountKey: string]: IamConfig[] }, +) { + const orgUnits = config.getOrganizationalUnits(); + const ouConfig = orgUnits.find(([orgName, _]) => orgName === account.ou); + if (!ouConfig) { + return; + } + const iamConfig = ouConfig[1].iam; + if (!iamConfig) { + return; + } + + if (!accountsIamConfig[account.key]) { + accountsIamConfig[account.key] = [iamConfig]; + } else { + const iamConfigs = accountsIamConfig[account.key]; + iamConfigs.push(iamConfig); + accountsIamConfig[account.key] = iamConfigs; + } +} + +export async function saveIamUsers( + iamConfigs: IamConfig[], + outputs: StackOutput[], + ssm: SSM, + accountKey: string, + acceleratorPrefix: string, + users: OutputUtilGenericType[], +): Promise { + const userIndices = users.flatMap(r => r.index) || []; + console.log('userIndices', userIndices); + let userMaxIndex = userIndices.length === 0 ? 0 : Math.max(...userIndices); + const updatedUsers: OutputUtilGenericType[] = []; + + for (const iamConfig of iamConfigs) { + if (!iamConfig || !iamConfig.users) { + continue; + } + + const userIds = iamConfig.users.flatMap(u => u['user-ids']); + console.log('userIds', userIds); + for (const user of userIds) { + const userOutput = IamUserOutputFinder.tryFindOneByName({ + outputs, + accountKey, + userKey: 'IamAccountUser', + userName: user, + }); + console.log('userOutput', userOutput); + if (!userOutput) { + console.warn(`Didn't find IAM User "${user}" in output`); + continue; + } + + let currentIndex: number; + const previousGroupIndexDetails = users.findIndex(p => p.name === userOutput.userName); + if (previousGroupIndexDetails >= 0) { + currentIndex = users[previousGroupIndexDetails].index; + console.log(`skipping creation of user ${userOutput.userName} in SSM`); + } else { + currentIndex = ++userMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/ident/user/${currentIndex}/name`, `${userOutput.userName}`); + await ssm.putParameter(`/${acceleratorPrefix}/ident/user/${currentIndex}/arn`, userOutput.userArn); + users.push({ + name: userOutput.userName, + index: currentIndex, + }); + } + updatedUsers.push({ + index: currentIndex, + name: userOutput.userName, + }); + } + } + return updatedUsers; +} + +export async function saveIamGroups( + iamConfigs: IamConfig[], + outputs: StackOutput[], + ssm: SSM, + accountKey: string, + acceleratorPrefix: string, + groups: OutputUtilGenericType[], +): Promise { + const groupIndices = groups.flatMap(r => r.index) || []; + console.log('groupIndices', groupIndices); + let policyMaxIndex = groupIndices.length === 0 ? 0 : Math.max(...groupIndices); + const updatedGroups: OutputUtilGenericType[] = []; + + for (const iamConfig of iamConfigs) { + if (!iamConfig || !iamConfig.users) { + continue; + } + + const groupIds = iamConfig.users.flatMap(u => u.group); + console.log('groupIds', groupIds); + for (const group of groupIds) { + const groupOutput = IamGroupOutputFinder.tryFindOneByName({ + outputs, + accountKey, + groupKey: 'IamAccountGroup', + groupName: group, + }); + if (!groupOutput) { + console.warn(`Didn't find IAM user group "${group}" in output`); + continue; + } + + let currentIndex: number; + const previousGroupIndexDetails = groups.findIndex(p => p.name === groupOutput.groupName); + if (previousGroupIndexDetails >= 0) { + currentIndex = groups[previousGroupIndexDetails].index; + console.log(`skipping creation of group ${groupOutput.groupName} in SSM`); + } else { + currentIndex = ++policyMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/ident/group/${currentIndex}/name`, `${groupOutput.groupName}`); + await ssm.putParameter(`/${acceleratorPrefix}/ident/group/${currentIndex}/arn`, groupOutput.groupArn); + groups.push({ + name: groupOutput.groupName, + index: currentIndex, + }); + } + updatedGroups.push({ + index: currentIndex, + name: groupOutput.groupName, + }); + } + } + return updatedGroups; +} + +export async function saveIamPolicy( + iamConfigs: IamConfig[], + outputs: StackOutput[], + ssm: SSM, + accountKey: string, + acceleratorPrefix: string, + policies: OutputUtilGenericType[], +): Promise { + const policyIndices = policies.flatMap(r => r.index) || []; + console.log('policyIndices', policyIndices); + let policyMaxIndex = policyIndices.length === 0 ? 0 : Math.max(...policyIndices); + const updatedPolicies: OutputUtilGenericType[] = []; + + for (const iamConfig of iamConfigs) { + if (!iamConfig || !iamConfig.policies) { + continue; + } + + const policyIds = iamConfig.policies.flatMap(p => p['policy-name']); + console.log('policyIds', policyIds); + for (const policy of policyIds) { + const policyOutput = IamPolicyOutputFinder.tryFindOneByName({ + outputs, + accountKey, + policyKey: 'IamCustomerManagedPolicy', + policyName: policy, + }); + if (!policyOutput) { + console.warn(`Didn't find IAM Policy "${policy}" in output`); + continue; + } + let currentIndex: number; + const previousPolicyIndexDetails = policies.findIndex(p => p.name === policyOutput.policyName); + if (previousPolicyIndexDetails >= 0) { + currentIndex = policies[previousPolicyIndexDetails].index; + console.log(`skipping creation of policy ${policyOutput.policyName} in SSM`); + } else { + currentIndex = ++policyMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/name`, `${policyOutput.policyName}`); + await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/arn`, policyOutput.policyArn); + policies.push({ + name: policyOutput.policyName, + index: currentIndex, + }); + } + updatedPolicies.push({ + index: currentIndex, + name: policyOutput.policyName, + }); + } + } + return updatedPolicies; +} + +export async function saveIamRoles( + iamConfigs: IamConfig[], + outputs: StackOutput[], + ssm: SSM, + accountKey: string, + acceleratorPrefix: string, + roles: OutputUtilGenericType[], +): Promise { + const roleIndices = roles.flatMap(r => r.index) || []; + console.log('roleIndices', roleIndices); + let rolesMaxIndex = roleIndices.length === 0 ? 0 : Math.max(...roleIndices); + const updatedRoles: OutputUtilGenericType[] = []; + + + for (const iamConfig of iamConfigs) { + if (!iamConfig || !iamConfig.roles) { + continue; + } + + const roleIds = iamConfig.roles.flatMap(r => r.role); + console.log('roleIds', roleIds); + for (const role of roleIds) { + const roleOutput = IamRoleNameOutputFinder.tryFindOneByName({ + outputs, + accountKey, + roleKey: 'IamAccountRole', + roleName: role, + }); + if (!roleOutput) { + console.warn(`Didn't find IAM Role "${role}" in output`); + continue; + } + + let currentIndex: number; + const previousRoleIndexDetails = roles.findIndex(p => p.name === roleOutput.roleName); + if (previousRoleIndexDetails >= 0) { + currentIndex = roles[previousRoleIndexDetails].index; + console.log(`skipping creation of role ${roleOutput.roleName} in SSM`); + } else { + currentIndex = ++rolesMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/ident/role/${currentIndex}/name`, `${roleOutput.roleName}`); + await ssm.putParameter(`/${acceleratorPrefix}/ident/role/${currentIndex}/arn`, roleOutput.roleArn); + roles.push({ + name: roleOutput.roleName, + index: currentIndex, + }); + } + updatedRoles.push({ + index: currentIndex, + name: roleOutput.roleName, + }); + } + } + return updatedRoles; +} \ No newline at end of file diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index c36db3a30..0a77d1a1f 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -5,6 +5,7 @@ import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { saveNetworkOutputs } from './network-outputs'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { saveIamOutputs } from './iam-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -43,19 +44,40 @@ export const handler = async (input: SaveOutputsToSsmInput) => { filePath: configFilePath, commitId: configCommitId, }); + + const globalRegions = config["global-options"]["additional-global-output-regions"]; + const smRegion = config["global-options"]["aws-org-master"].region; + + // TODO preparing list of regions to create IAM parameters + const iamRegions = [...globalRegions, smRegion]; + + if (iamRegions.includes(region)) { + // Store Identity Outputs to SSM Parameter Store + await saveIamOutputs({ + acceleratorPrefix, + config, + dynamodb, + outputsTableName, + assumeRoleName, + account, + region, + outputUtilsTableName, + }); + } + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); const ssm = new SSM(credentials, region); // Store Network Outputs to SSM Parameter Store - await saveNetworkOutputs({ - acceleratorPrefix, - config, - dynamodb, - outputsTableName, - ssm, - account, - region, - outputUtilsTableName, - }); + // await saveNetworkOutputs({ + // acceleratorPrefix, + // config, + // dynamodb, + // outputsTableName, + // ssm, + // account, + // region, + // outputUtilsTableName, + // }); return { status: 'SUCCESS', diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index c97dd4e77..1380a89d8 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -3,6 +3,7 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { getItemInput, getUpdateItemInput } from '../utils/dynamodb-requests'; export interface SaveOutputsInput { acceleratorPrefix: string; @@ -10,7 +11,8 @@ export interface SaveOutputsInput { dynamodb: DynamoDB; config: AcceleratorConfig; account: Account; - ssm: SSM; + // ssm: SSM; + assumeRoleName: string; region: string; outputUtilsTableName: string; } @@ -37,3 +39,20 @@ export async function getOutputUtil(tableName: string, key: string, dynamodb: Dy } return JSON.parse(outputUtils.S); } + +export async function getIamSsmOutput(tableName: string, key: string, dynamodb: DynamoDB): Promise { + const cfnOutputs = await dynamodb.getItem(getItemInput(tableName, key)); + if (!cfnOutputs.Item) { + return; + } + return cfnOutputs.Item.value.S!; +} + +export async function saveIndexOutput( + tableName: string, + key: string, + value: string, + dynamodb: DynamoDB, +): Promise { + await dynamodb.updateItem(getUpdateItemInput(tableName, key, value)); +} \ No newline at end of file diff --git a/src/deployments/cdk/src/common/iam-assets.ts b/src/deployments/cdk/src/common/iam-assets.ts index dc2c90d0a..7a0eab020 100644 --- a/src/deployments/cdk/src/common/iam-assets.ts +++ b/src/deployments/cdk/src/common/iam-assets.ts @@ -10,6 +10,7 @@ import { import { Account, getAccountId } from '../utils/accounts'; import { IBucket } from '@aws-cdk/aws-s3'; import { createPolicyName } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; +import { CfnIamPolicyOutput, CfnIamRoleOutput, CfnIamUserOutput, CfnIamGroupOutput } from '../deployments/iam'; export interface IamAssetsProps { accountKey: string; @@ -47,6 +48,11 @@ export class IamAssets extends cdk.Construct { ); } customerManagedPolicies[policyName] = iamPolicy; + new CfnIamPolicyOutput(this, `IamPolicy${policyName}Output`, { + policyName: iamPolicy.managedPolicyName, + policyArn: iamPolicy.managedPolicyArn, + policyKey: 'IamCustomerManagedPolicy', + }); }; // method to create IAM User & Group @@ -56,6 +62,12 @@ export class IamAssets extends cdk.Construct { managedPolicies: policies.map(x => iam.ManagedPolicy.fromAwsManagedPolicyName(x)), }); + new CfnIamGroupOutput(this, `IamGroup${groupName}Output`, { + groupName: iamGroup.groupName, + groupArn: iamGroup.groupArn, + groupKey: 'IamAccountGroup', + }); + for (const userId of userIds) { const iamUser = new iam.User(this, `IAM-User-${userId}-${accountKey}`, { userName: userId, @@ -63,6 +75,12 @@ export class IamAssets extends cdk.Construct { groups: [iamGroup], permissionsBoundary: customerManagedPolicies[boundaryPolicy], }); + + new CfnIamUserOutput(this, `IamUser${userId}Output`, { + userName: iamUser.userName, + userArn: iamUser.userArn, + userKey: 'IamAccountUser', + }); } }; @@ -133,6 +151,11 @@ export class IamAssets extends cdk.Construct { resources: [logBucket.arnForObjects('*')], }), ); + new CfnIamPolicyOutput(this, `IamSsmPolicyOutput`, { + policyName: iamSSMLogArchiveAccessPolicy.managedPolicyName, + policyArn: iamSSMLogArchiveAccessPolicy.managedPolicyArn, + policyKey: 'IamSsmAccessPolicy', + }); return iamSSMLogArchiveAccessPolicy; }; @@ -195,6 +218,12 @@ export class IamAssets extends cdk.Construct { }); } + new CfnIamRoleOutput(this, `IamRole${iamRole.role}Output`, { + roleName: role.roleName, + roleArn: role.roleArn, + roleKey: 'IamAccountRole', + }); + if (iamRole['ssm-log-archive-access'] && ssmLogArchivePolicy) { role.addManagedPolicy(ssmLogArchivePolicy); } diff --git a/src/deployments/cdk/src/deployments/iam/outputs.ts b/src/deployments/cdk/src/deployments/iam/outputs.ts index d16889186..e2b6e0b3f 100644 --- a/src/deployments/cdk/src/deployments/iam/outputs.ts +++ b/src/deployments/cdk/src/deployments/iam/outputs.ts @@ -3,11 +3,18 @@ import * as iam from '@aws-cdk/aws-iam'; import { createFixedSecretName } from '@aws-accelerator/common-outputs/src/secrets'; import { createCfnStructuredOutput } from '../../common/structured-output'; -import { IamRoleOutput } from '@aws-accelerator/common-outputs/src/iam-role'; +import { IamRoleOutput, IamPolicyOutput } from '@aws-accelerator/common-outputs/src/iam-role'; +import { IamUserOutput, IamGroupOutput } from '@aws-accelerator/common-outputs/src/iam-users'; import { AccountStack } from '../../common/account-stacks'; export const CfnIamRoleOutput = createCfnStructuredOutput(IamRoleOutput); +export const CfnIamPolicyOutput = createCfnStructuredOutput(IamPolicyOutput); + +export const CfnIamUserOutput = createCfnStructuredOutput(IamUserOutput); + +export const CfnIamGroupOutput = createCfnStructuredOutput(IamGroupOutput); + export function createIamUserPasswordSecretName({ acceleratorPrefix, accountKey, diff --git a/src/lib/common-config/src/index.ts b/src/lib/common-config/src/index.ts index 9eacfd969..ddaa1b39c 100644 --- a/src/lib/common-config/src/index.ts +++ b/src/lib/common-config/src/index.ts @@ -764,6 +764,7 @@ export const GlobalOptionsConfigType = t.interface({ 'workloadaccounts-param-filename': t.string, 'vpc-flow-logs': VpcFlowLogsConfigType, 'additional-cwl-regions': fromNullable(t.record(t.string, AdditionalCwlRegionType), {}), + 'additional-global-output-regions': fromNullable(t.array(t.string), []), cloudwatch: optional( t.interface({ metrics: t.array(CloudWatchMetricFiltersConfigType), diff --git a/src/lib/common-outputs/src/iam-role.ts b/src/lib/common-outputs/src/iam-role.ts index dcc3ddec2..a302d0db2 100644 --- a/src/lib/common-outputs/src/iam-role.ts +++ b/src/lib/common-outputs/src/iam-role.ts @@ -21,3 +21,32 @@ export const IamRoleOutputFinder = createStructuredOutputFinder(IamRoleOutput, f predicate: o => o.roleKey === props.roleKey, }), })); + +export const IamRoleNameOutputFinder = createStructuredOutputFinder(IamRoleOutput, finder => ({ + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; roleName: string; roleKey?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + predicate: o => o.roleKey === props.roleKey && o.roleName === props.roleName, + }), +})); + +export const IamPolicyOutput = t.interface( + { + policyName: t.string, + policyArn: t.string, + policyKey: t.string, + }, + 'IamPolicy', +); + +export type IamPolicyOutput = t.TypeOf; + +export const IamPolicyOutputFinder = createStructuredOutputFinder(IamPolicyOutput, finder => ({ + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; policyKey?: string; policyName?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + predicate: o => o.policyKey === props.policyKey && o.policyName == props.policyName, + }), +})); diff --git a/src/lib/common-outputs/src/iam-users.ts b/src/lib/common-outputs/src/iam-users.ts new file mode 100644 index 000000000..e85d3f0ed --- /dev/null +++ b/src/lib/common-outputs/src/iam-users.ts @@ -0,0 +1,43 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; +import { StackOutput } from './stack-output'; + +export const IamUserOutput = t.interface( + { + userName: t.string, + userArn: t.string, + userKey: t.string, + }, + 'IamUser', +); + +export type IamUserOutput = t.TypeOf; + +export const IamUserOutputFinder = createStructuredOutputFinder(IamUserOutput, finder => ({ + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; userKey?: string; userName?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + predicate: o => o.userKey === props.userKey && o.userName === props.userName, + }), +})); + +export const IamGroupOutput = t.interface( + { + groupName: t.string, + groupArn: t.string, + groupKey: t.string, + }, + 'IamGroup', +); + +export type IamGroupOutput = t.TypeOf; + +export const IamGroupOutputFinder = createStructuredOutputFinder(IamGroupOutput, finder => ({ + tryFindOneByName: (props: { outputs: StackOutput[]; accountKey: string; groupKey?: string; groupName?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + predicate: o => o.groupKey === props.groupKey && o.groupName === props.groupName, + }), +})); From e927e83983fd4a13e5687881485c0123fb985bb3 Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 22 Sep 2020 21:22:06 +0530 Subject: [PATCH 10/25] moved ssm into category --- .../runtime/src/save-outputs-to-ssm/network-outputs.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 36555171c..b361e55ab 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -10,6 +10,7 @@ import { ResolvedVpcConfig, SecurityGroupConfig, SubnetConfig } from '@aws-accel import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { getUpdateValueInput } from '../utils/dynamodb-requests'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; interface OutputUtilSubnet extends OutputUtilGenericType { azs: string[]; @@ -44,7 +45,7 @@ function getIndex(input: T[], searchName: string) { * @returns void */ export async function saveNetworkOutputs(props: SaveOutputsInput) { - const { acceleratorPrefix, account, config, dynamodb, outputsTableName, ssm, region, outputUtilsTableName } = props; + const { acceleratorPrefix, account, config, dynamodb, outputsTableName, assumeRoleName, region, outputUtilsTableName } = props; const oldNetworkOutputUtils = await getOutputUtil(outputUtilsTableName, `${account.key}-${region}-network`, dynamodb); // Existing index check happens on this variable let networkOutputUtils: OutputUtilNetwork; @@ -94,6 +95,11 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { } const lvpcIndices = networkOutputUtils.vpcs.filter(lv => lv.type === 'lvpc').flatMap(v => v.index) || []; let lvpcMaxIndex = lvpcIndices.length === 0 ? 0 : Math.max(...lvpcIndices); + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + for (const resolvedVpcConfig of localVpcConfigs) { let currentIndex: number; const previousIndex = networkOutputUtils.vpcs.findIndex( From c5e24f4fb1e9cd730be1c74c9f9a64336fcecaae Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 22 Sep 2020 21:53:31 +0530 Subject: [PATCH 11/25] added removal of ssm parameters --- .../src/save-outputs-to-ssm/iam-utils.ts | 82 ++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts index 680873c14..b7f917b55 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -60,6 +60,7 @@ export async function saveIamUsers( console.log('userIndices', userIndices); let userMaxIndex = userIndices.length === 0 ? 0 : Math.max(...userIndices); const updatedUsers: OutputUtilGenericType[] = []; + const removalObjects: OutputUtilGenericType[] = [...(users || [])]; for (const iamConfig of iamConfigs) { if (!iamConfig || !iamConfig.users) { @@ -75,7 +76,6 @@ export async function saveIamUsers( userKey: 'IamAccountUser', userName: user, }); - console.log('userOutput', userOutput); if (!userOutput) { console.warn(`Didn't find IAM User "${user}" in output`); continue; @@ -99,6 +99,25 @@ export async function saveIamUsers( index: currentIndex, name: userOutput.userName, }); + + const removalIndex = removalObjects.findIndex( + p => p.name === userOutput.userName, + ); + if (removalIndex != -1) { + removalObjects.splice(removalIndex, 1); + } + } + } + + for (const removeObject of removalObjects || []) { + const removalUsers = ([ + `/${acceleratorPrefix}/ident/user/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/user/${removeObject.index}/arn`, + ]) + .flatMap(s => s); + + while (removalUsers.length > 0) { + await ssm.deleteParameters(removalUsers.splice(0, 10)); } } return updatedUsers; @@ -116,6 +135,7 @@ export async function saveIamGroups( console.log('groupIndices', groupIndices); let policyMaxIndex = groupIndices.length === 0 ? 0 : Math.max(...groupIndices); const updatedGroups: OutputUtilGenericType[] = []; + const removalObjects: OutputUtilGenericType[] = [...(groups || [])]; for (const iamConfig of iamConfigs) { if (!iamConfig || !iamConfig.users) { @@ -154,6 +174,25 @@ export async function saveIamGroups( index: currentIndex, name: groupOutput.groupName, }); + + const removalIndex = removalObjects.findIndex( + p => p.name === groupOutput.groupName, + ); + if (removalIndex != -1) { + removalObjects.splice(removalIndex, 1); + } + } + } + + for (const removeObject of removalObjects || []) { + const removalGroups = ([ + `/${acceleratorPrefix}/ident/group/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/group/${removeObject.index}/arn`, + ]) + .flatMap(s => s); + + while (removalGroups.length > 0) { + await ssm.deleteParameters(removalGroups.splice(0, 10)); } } return updatedGroups; @@ -171,6 +210,7 @@ export async function saveIamPolicy( console.log('policyIndices', policyIndices); let policyMaxIndex = policyIndices.length === 0 ? 0 : Math.max(...policyIndices); const updatedPolicies: OutputUtilGenericType[] = []; + const removalObjects: OutputUtilGenericType[] = [...(policies || [])]; for (const iamConfig of iamConfigs) { if (!iamConfig || !iamConfig.policies) { @@ -208,6 +248,25 @@ export async function saveIamPolicy( index: currentIndex, name: policyOutput.policyName, }); + + const removalIndex = removalObjects.findIndex( + p => p.name === policyOutput.policyName, + ); + if (removalIndex != -1) { + removalObjects.splice(removalIndex, 1); + } + } + } + + for (const removeObject of removalObjects || []) { + const removalPolicies = ([ + `/${acceleratorPrefix}/ident/policy/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/policy/${removeObject.index}/arn`, + ]) + .flatMap(s => s); + + while (removalPolicies.length > 0) { + await ssm.deleteParameters(removalPolicies.splice(0, 10)); } } return updatedPolicies; @@ -225,7 +284,7 @@ export async function saveIamRoles( console.log('roleIndices', roleIndices); let rolesMaxIndex = roleIndices.length === 0 ? 0 : Math.max(...roleIndices); const updatedRoles: OutputUtilGenericType[] = []; - + const removalObjects: OutputUtilGenericType[] = [...(roles || [])]; for (const iamConfig of iamConfigs) { if (!iamConfig || !iamConfig.roles) { @@ -264,6 +323,25 @@ export async function saveIamRoles( index: currentIndex, name: roleOutput.roleName, }); + + const removalIndex = removalObjects.findIndex( + p => p.name === roleOutput.roleName, + ); + if (removalIndex != -1) { + removalObjects.splice(removalIndex, 1); + } + } + } + + for (const removeObject of removalObjects || []) { + const removalRoles = ([ + `/${acceleratorPrefix}/ident/role/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/role/${removeObject.index}/arn`, + ]) + .flatMap(s => s); + + while (removalRoles.length > 0) { + await ssm.deleteParameters(removalRoles.splice(0, 10)); } } return updatedRoles; From 1e5652adcadc53a68e90ca2eb7ebf062f70a5cae Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 23 Sep 2020 08:42:41 +0530 Subject: [PATCH 12/25] Ignore updation for ssm params --- .../save-outputs-to-ssm/network-outputs.ts | 92 +++++++++++++------ 1 file changed, 62 insertions(+), 30 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 36555171c..7606a00d3 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -26,12 +26,10 @@ interface OutputUtilNetwork { vpcs?: OutputUtilVpc[]; } -function getIndex(input: T[], searchName: string) { - console.log(typeof input); -} /** * Outputs for network related deployments will be found in following phases - * + * - Phase-1 + * - Phase-2 */ /** @@ -113,7 +111,7 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { acceleratorPrefix, vpcPrefix: 'lvpc', account, - vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, + vpcUtil: previousIndex >= 0 ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { newNetworkOutputs.vpcs.push(vpcResult); @@ -148,7 +146,7 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { acceleratorPrefix, vpcPrefix: 'vpc', account, - vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, + vpcUtil: previousIndex >= 0 ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { @@ -195,7 +193,7 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { account, sgOutputs: vpcSgOutputs?.securityGroupIds, sharedVpc: true, - vpcUtil: previousIndex ? networkOutputUtils.vpcs[previousIndex] : undefined, + vpcUtil: previousIndex >= 0 ? networkOutputUtils.vpcs[previousIndex] : undefined, }); if (vpcResult) { @@ -270,9 +268,11 @@ async function saveVpcOutputs(props: { const { acceleratorPrefix, account, index, outputs, resolvedVpcConfig, ssm, vpcPrefix, sgOutputs, sharedVpc } = props; const { accountKey, vpcConfig } = resolvedVpcConfig; let vpcUtil: OutputUtilVpc; + let updateRequired = false; if (props.vpcUtil) { vpcUtil = props.vpcUtil; } else { + updateRequired = true; vpcUtil = { index, name: vpcConfig.name, @@ -290,9 +290,11 @@ async function saveVpcOutputs(props: { console.warn(`VPC "${vpcConfig.name}" in account "${accountKey}" is not created`); return; } - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); - await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); + if (updateRequired) { + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/name`, `${vpcOutput.vpcName}_vpc`); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/id`, vpcOutput.vpcId); + await ssm.putParameter(`/${acceleratorPrefix}/network/${vpcPrefix}/${index}/cidr`, vpcOutput.cidrBlock); + } let subnetsConfig = vpcConfig.subnets; if (sharedVpc) { subnetsConfig = vpcConfig.subnets?.filter( @@ -369,19 +371,32 @@ export async function saveSecurityGroups(props: { console.warn(`Didn't find SecurityGroup "${sgConfig.name}" in output`); continue; } - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/name`, - `${sgConfig.name}_sg`, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/id`, - sgOutput.securityGroupId, - ); + if (previousIndex < 0) { + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/name`, + `${sgConfig.name}_sg`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${currentIndex}/id`, + sgOutput.securityGroupId, + ); + } + const removalIndex = removalObjects?.findIndex(r => r.name === sgConfig.name); if (removalIndex >= 0) { removalObjects?.splice(removalIndex, 1); } } + + const removeNames = removalObjects + .map(sg => [ + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sg.index}/name`, + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/sg/${sg.index}/id`, + ]) + .flatMap(s => s); + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } return updatedObjects; } @@ -419,23 +434,40 @@ export async function saveSubnets(props: { console.warn(`Didn't find subnet "${subnetConfig.name}" in output`); continue; } - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetDef.az}/name`, - `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/id`, - subnetOutput.subnetId, - ); - await ssm.putParameter( - `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/cidr`, - subnetOutput.cidrBlock, - ); + if (previousIndex < 0 || (previousIndex >= 0 && !subnetsUtil[previousIndex].azs.includes(subnetDef.az))) { + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetDef.az}/name`, + `${subnetOutput.subnetName}_${vpcName}_az${subnetOutput.az}_net`, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/id`, + subnetOutput.subnetId, + ); + await ssm.putParameter( + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${currentIndex}/az${subnetOutput.az}/cidr`, + subnetOutput.cidrBlock, + ); + } } const removalIndex = removalObjects?.findIndex(s => s.name === subnetConfig.name); if (removalIndex >= 0) { removalObjects?.splice(removalIndex, 1); } } + + const removeNames = removalObjects + .map(sn => + sn.azs.map(snz => [ + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${sn.index}/az${snz}/name`, + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${sn.index}/az${snz}/id`, + `/${acceleratorPrefix}/network/${vpcPrefix}/${vpcIndex}/net/${sn.index}/az${snz}/cidr`, + ]), + ) + .flatMap(azSn => azSn) + .flatMap(sn => sn); + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } + return updatedObjects; } From 86e662b7f30e28dd17aeddf2ea243b4e46004148 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 23 Sep 2020 09:35:33 +0530 Subject: [PATCH 13/25] Updates in utils --- .../src/save-outputs-to-ssm/iam-outputs.ts | 13 ++++- .../src/save-outputs-to-ssm/iam-utils.ts | 54 ++++++++----------- .../runtime/src/save-outputs-to-ssm/index.ts | 26 +++++---- .../save-outputs-to-ssm/network-outputs.ts | 41 +++++++------- .../runtime/src/save-outputs-to-ssm/utils.ts | 24 +++++++-- src/lib/common/src/aws/dynamodb.ts | 10 ++-- 6 files changed, 92 insertions(+), 76 deletions(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts index b0aa0bbc6..19da0668a 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts @@ -34,7 +34,16 @@ interface IamOutput { * @returns void */ export async function saveIamOutputs(props: SaveOutputsInput) { - const { acceleratorPrefix, account, config, dynamodb, outputsTableName, assumeRoleName, region, outputUtilsTableName } = props; + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; const accountConfig = config.getMandatoryAccountConfigs().find(([accountKey, _]) => accountKey === account.key); const accountsIam: { [accountKey: string]: IamConfig[] } = {}; @@ -51,7 +60,7 @@ export async function saveIamOutputs(props: SaveOutputsInput) { return; } - const smRegion = config["global-options"]["aws-org-master"].region; + const smRegion = config['global-options']['aws-org-master'].region; const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${smRegion}-1`, dynamodb); const ssmOutputs = await getIamSsmOutput(outputUtilsTableName, `${account.key}-${region}-identity`, dynamodb); // console.log('ssmOutputs', ssmOutputs); diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts index b7f917b55..f766f59ba 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -100,9 +100,7 @@ export async function saveIamUsers( name: userOutput.userName, }); - const removalIndex = removalObjects.findIndex( - p => p.name === userOutput.userName, - ); + const removalIndex = removalObjects.findIndex(p => p.name === userOutput.userName); if (removalIndex != -1) { removalObjects.splice(removalIndex, 1); } @@ -110,11 +108,10 @@ export async function saveIamUsers( } for (const removeObject of removalObjects || []) { - const removalUsers = ([ - `/${acceleratorPrefix}/ident/user/${removeObject.index}/name`, - `/${acceleratorPrefix}/ident/user/${removeObject.index}/arn`, - ]) - .flatMap(s => s); + const removalUsers = [ + `/${acceleratorPrefix}/ident/user/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/user/${removeObject.index}/arn`, + ].flatMap(s => s); while (removalUsers.length > 0) { await ssm.deleteParameters(removalUsers.splice(0, 10)); @@ -175,9 +172,7 @@ export async function saveIamGroups( name: groupOutput.groupName, }); - const removalIndex = removalObjects.findIndex( - p => p.name === groupOutput.groupName, - ); + const removalIndex = removalObjects.findIndex(p => p.name === groupOutput.groupName); if (removalIndex != -1) { removalObjects.splice(removalIndex, 1); } @@ -185,11 +180,10 @@ export async function saveIamGroups( } for (const removeObject of removalObjects || []) { - const removalGroups = ([ - `/${acceleratorPrefix}/ident/group/${removeObject.index}/name`, - `/${acceleratorPrefix}/ident/group/${removeObject.index}/arn`, - ]) - .flatMap(s => s); + const removalGroups = [ + `/${acceleratorPrefix}/ident/group/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/group/${removeObject.index}/arn`, + ].flatMap(s => s); while (removalGroups.length > 0) { await ssm.deleteParameters(removalGroups.splice(0, 10)); @@ -249,9 +243,7 @@ export async function saveIamPolicy( name: policyOutput.policyName, }); - const removalIndex = removalObjects.findIndex( - p => p.name === policyOutput.policyName, - ); + const removalIndex = removalObjects.findIndex(p => p.name === policyOutput.policyName); if (removalIndex != -1) { removalObjects.splice(removalIndex, 1); } @@ -259,11 +251,10 @@ export async function saveIamPolicy( } for (const removeObject of removalObjects || []) { - const removalPolicies = ([ - `/${acceleratorPrefix}/ident/policy/${removeObject.index}/name`, - `/${acceleratorPrefix}/ident/policy/${removeObject.index}/arn`, - ]) - .flatMap(s => s); + const removalPolicies = [ + `/${acceleratorPrefix}/ident/policy/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/policy/${removeObject.index}/arn`, + ].flatMap(s => s); while (removalPolicies.length > 0) { await ssm.deleteParameters(removalPolicies.splice(0, 10)); @@ -324,9 +315,7 @@ export async function saveIamRoles( name: roleOutput.roleName, }); - const removalIndex = removalObjects.findIndex( - p => p.name === roleOutput.roleName, - ); + const removalIndex = removalObjects.findIndex(p => p.name === roleOutput.roleName); if (removalIndex != -1) { removalObjects.splice(removalIndex, 1); } @@ -334,15 +323,14 @@ export async function saveIamRoles( } for (const removeObject of removalObjects || []) { - const removalRoles = ([ - `/${acceleratorPrefix}/ident/role/${removeObject.index}/name`, - `/${acceleratorPrefix}/ident/role/${removeObject.index}/arn`, - ]) - .flatMap(s => s); + const removalRoles = [ + `/${acceleratorPrefix}/ident/role/${removeObject.index}/name`, + `/${acceleratorPrefix}/ident/role/${removeObject.index}/arn`, + ].flatMap(s => s); while (removalRoles.length > 0) { await ssm.deleteParameters(removalRoles.splice(0, 10)); } } return updatedRoles; -} \ No newline at end of file +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 0a77d1a1f..97f34cf9e 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -45,8 +45,8 @@ export const handler = async (input: SaveOutputsToSsmInput) => { commitId: configCommitId, }); - const globalRegions = config["global-options"]["additional-global-output-regions"]; - const smRegion = config["global-options"]["aws-org-master"].region; + const globalRegions = config['global-options']['additional-global-output-regions']; + const smRegion = config['global-options']['aws-org-master'].region; // TODO preparing list of regions to create IAM parameters const iamRegions = [...globalRegions, smRegion]; @@ -65,19 +65,17 @@ export const handler = async (input: SaveOutputsToSsmInput) => { }); } - const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); - const ssm = new SSM(credentials, region); // Store Network Outputs to SSM Parameter Store - // await saveNetworkOutputs({ - // acceleratorPrefix, - // config, - // dynamodb, - // outputsTableName, - // ssm, - // account, - // region, - // outputUtilsTableName, - // }); + await saveNetworkOutputs({ + acceleratorPrefix, + config, + dynamodb, + outputsTableName, + assumeRoleName, + account, + region, + outputUtilsTableName, + }); return { status: 'SUCCESS', diff --git a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts index 7b88dd725..12bbecb2c 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/network-outputs.ts @@ -1,5 +1,5 @@ import { getStackJsonOutput, StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { getOutput, OutputUtilGenericType, SaveOutputsInput, getOutputUtil } from './utils'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, getIndexOutput, saveIndexOutput } from './utils'; import { SecurityGroupsOutput, VpcOutputFinder, @@ -9,7 +9,6 @@ import { import { ResolvedVpcConfig, SecurityGroupConfig, SubnetConfig } from '@aws-accelerator/common-config/'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -import { getUpdateValueInput } from '../utils/dynamodb-requests'; import { STS } from '@aws-accelerator/common/src/aws/sts'; interface OutputUtilSubnet extends OutputUtilGenericType { @@ -43,8 +42,21 @@ interface OutputUtilNetwork { * @returns void */ export async function saveNetworkOutputs(props: SaveOutputsInput) { - const { acceleratorPrefix, account, config, dynamodb, outputsTableName, assumeRoleName, region, outputUtilsTableName } = props; - const oldNetworkOutputUtils = await getOutputUtil(outputUtilsTableName, `${account.key}-${region}-network`, dynamodb); + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; + const oldNetworkOutputUtils = await getIndexOutput( + outputUtilsTableName, + `${account.key}-${region}-network`, + dynamodb, + ); // Existing index check happens on this variable let networkOutputUtils: OutputUtilNetwork; if (oldNetworkOutputUtils) { @@ -215,21 +227,12 @@ export async function saveNetworkOutputs(props: SaveOutputsInput) { } } - const updateExpression = getUpdateValueInput([ - { - key: 'v', - name: 'outputValue', - type: 'S', - value: JSON.stringify(newNetworkOutputs), - }, - ]); - await dynamodb.updateItem({ - TableName: outputUtilsTableName, - Key: { - id: { S: `${account.key}-${region}-network` }, - }, - ...updateExpression, - }); + await saveIndexOutput( + outputUtilsTableName, + `${account.key}-${region}-network`, + JSON.stringify(newNetworkOutputs), + dynamodb, + ); for (const removeObject of removalObjects.vpcs || []) { const removalSgs = removeObject.securityGroups .map(sg => [ diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 1380a89d8..539ceea48 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -3,7 +3,7 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -import { getItemInput, getUpdateItemInput } from '../utils/dynamodb-requests'; +import { getItemInput, getUpdateItemInput, getUpdateValueInput } from '../utils/dynamodb-requests'; export interface SaveOutputsInput { acceleratorPrefix: string; @@ -32,8 +32,8 @@ export async function getOutput(tableName: string, key: string, dynamodb: Dynamo return outputs; } -export async function getOutputUtil(tableName: string, key: string, dynamodb: DynamoDB) { - const outputUtils = await dynamodb.getOutputValue(tableName, key); +export async function getIndexOutput(tableName: string, key: string, dynamodb: DynamoDB) { + const outputUtils = await dynamodb.getOutputValue(tableName, key, 'value'); if (!outputUtils || !outputUtils.S) { return; } @@ -54,5 +54,19 @@ export async function saveIndexOutput( value: string, dynamodb: DynamoDB, ): Promise { - await dynamodb.updateItem(getUpdateItemInput(tableName, key, value)); -} \ No newline at end of file + const updateExpression = getUpdateValueInput([ + { + key: 'v', + name: 'value', + type: 'S', + value: value, + }, + ]); + await dynamodb.updateItem({ + TableName: tableName, + Key: { + id: { S: key }, + }, + ...updateExpression, + }); +} diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 155181141..7912e55b4 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -61,15 +61,19 @@ export class DynamoDB { await throttlingBackOff(() => this.client.updateItem(props).promise()); } - async getOutputValue(tableName: string, key: string): Promise { + async getOutputValue( + tableName: string, + key: string, + keyName: string = 'outputValue', + ): Promise { const outputResponse = await this.getItem({ Key: { id: { S: key } }, TableName: tableName, - AttributesToGet: ['outputValue'], + AttributesToGet: [keyName], }); if (!outputResponse.Item) { return; } - return outputResponse.Item.outputValue; + return outputResponse.Item[keyName]; } } From 7520e090688a3ea939c469e00134eeffd22eb54b Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 23 Sep 2020 15:46:00 +0530 Subject: [PATCH 14/25] added kms and acm outputs --- .../src/deployments/certificates/outputs.ts | 26 ++++++++++++ .../src/deployments/certificates/step-1.ts | 7 +++- .../cdk/src/deployments/defaults/outputs.ts | 40 +++++++++++++++++++ .../cdk/src/deployments/defaults/shared.ts | 15 +++++-- .../cdk/src/deployments/defaults/step-1.ts | 20 ++++++++-- .../cdk/src/deployments/defaults/step-2.ts | 4 +- .../cdk/src/deployments/secrets/outputs.ts | 2 + .../cdk/src/deployments/secrets/step-1.ts | 2 + .../src/deployments/ssm/session-manager.ts | 11 ++++- .../src/core/secrets-container.ts | 8 +++- src/lib/common-outputs/src/central-bucket.ts | 2 + src/lib/common-outputs/src/ebs.ts | 23 +++++++++++ src/lib/common-outputs/src/ssm.ts | 20 ++++++++++ 13 files changed, 168 insertions(+), 12 deletions(-) create mode 100644 src/lib/common-outputs/src/ebs.ts diff --git a/src/deployments/cdk/src/deployments/certificates/outputs.ts b/src/deployments/cdk/src/deployments/certificates/outputs.ts index 5d781d379..255c678e2 100644 --- a/src/deployments/cdk/src/deployments/certificates/outputs.ts +++ b/src/deployments/cdk/src/deployments/certificates/outputs.ts @@ -1,3 +1,29 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from '@aws-accelerator/common-outputs/src/structured-output'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { createCfnStructuredOutput } from '../../common/structured-output'; + export function createCertificateSecretName(certificateName: string): string { return `accelerator/certificates/${certificateName}`; } + +export const AcmOutput = t.interface( + { + certificateName: t.string, + certificateArn: t.string, + }, + 'Acm', +); + +export type AcmOutput = t.TypeOf; + +export const CfnAcmOutput = createCfnStructuredOutput(AcmOutput); + +export const AcmOutputFinder = createStructuredOutputFinder(AcmOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); \ No newline at end of file diff --git a/src/deployments/cdk/src/deployments/certificates/step-1.ts b/src/deployments/cdk/src/deployments/certificates/step-1.ts index 9e5163d06..d8a4ad8e7 100644 --- a/src/deployments/cdk/src/deployments/certificates/step-1.ts +++ b/src/deployments/cdk/src/deployments/certificates/step-1.ts @@ -6,7 +6,7 @@ import * as c from '@aws-accelerator/common-config/src'; import { AcmImportCertificate } from '@aws-accelerator/custom-resource-acm-import-certificate'; import { AccountStacks } from '../../common/account-stacks'; import { pascalCase } from 'pascal-case'; -import { createCertificateSecretName } from './outputs'; +import { createCertificateSecretName, CfnAcmOutput } from './outputs'; export interface CertificatesStep1Props { accountStacks: AccountStacks; @@ -78,6 +78,11 @@ function createCertificate(props: { description: `Certificate ARN for certificate ${certificate.name}`, secretString: resource.certificateArn, }); + + new CfnAcmOutput(scope, `Cert${certificatePrettyName}Output`, { + certificateName: certificate.name, + certificateArn: resource.certificateArn, + }); } } diff --git a/src/deployments/cdk/src/deployments/defaults/outputs.ts b/src/deployments/cdk/src/deployments/defaults/outputs.ts index 216e86328..4444545be 100644 --- a/src/deployments/cdk/src/deployments/defaults/outputs.ts +++ b/src/deployments/cdk/src/deployments/defaults/outputs.ts @@ -7,6 +7,13 @@ import { AcceleratorConfig } from '@aws-accelerator/common-config/src'; import { Account } from '../../utils/accounts'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { StructuredOutput, createCfnStructuredOutput } from '../../common/structured-output'; +import { EbsKmsOutput } from '@aws-accelerator/common-outputs/src/ebs'; +import { SsmKmsOutput } from '@aws-accelerator/common-outputs/src/ssm'; +import { createStructuredOutputFinder } from '@aws-accelerator/common-outputs/src/structured-output'; + +export const CfnEbsKmsOutput = createCfnStructuredOutput(EbsKmsOutput); + +export const CfnSsmKmsOutput = createCfnStructuredOutput(SsmKmsOutput); export interface RegionalBucket extends s3.IBucket { region: string; @@ -37,6 +44,8 @@ const AccountBucketOutputType = t.interface( bucketArn: t.string, encryptionKeyArn: t.string, region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, }, 'AccountBucket', ); @@ -49,6 +58,8 @@ const LogBucketOutputType = t.interface( bucketArn: t.string, encryptionKeyArn: t.string, region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, }, 'LogBucket', ); @@ -61,6 +72,8 @@ const CentralBucketOutputType = t.interface( bucketArn: t.string, encryptionKeyArn: t.string, region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, }, 'CentralBucket', ); @@ -83,6 +96,33 @@ export const CfnLogBucketOutput = createCfnStructuredOutput(LogBucketOutputType) export const CfnCentralBucketOutput = createCfnStructuredOutput(CentralBucketOutputType); export const CfnAesBucketOutput = createCfnStructuredOutput(AesBucketOutputType); +export const AccountBucketOutputFinder = createStructuredOutputFinder(AccountBucketOutputType, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); + +export const LogBucketOutputTypeOutputFinder = createStructuredOutputFinder(LogBucketOutputType, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); + +export const CentralBucketOutputFinder = createStructuredOutputFinder(CentralBucketOutputType, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); + export namespace AccountBucketOutput { /** * Helper method to import the account buckets from different phases. It includes the log bucket. diff --git a/src/deployments/cdk/src/deployments/defaults/shared.ts b/src/deployments/cdk/src/deployments/defaults/shared.ts index 12b393838..c3ca89902 100644 --- a/src/deployments/cdk/src/deployments/defaults/shared.ts +++ b/src/deployments/cdk/src/deployments/defaults/shared.ts @@ -6,11 +6,17 @@ import { createEncryptionKeyName } from '@aws-accelerator/cdk-accelerator/src/co import { AccountStack } from '../../common/account-stacks'; import { overrideLogicalId } from '../../utils/cdk'; -export function createDefaultS3Key(props: { accountStack: AccountStack }): kms.Key { +export interface KmsDetails { + encryptionKey: kms.Key; + alias: string; +} + +export function createDefaultS3Key(props: { accountStack: AccountStack }): KmsDetails { const { accountStack } = props; + const keyAlias = createEncryptionKeyName('Bucket-Key'); const encryptionKey = new kms.Key(accountStack, 'DefaultKey', { - alias: 'alias/' + createEncryptionKeyName('Bucket-Key'), + alias: `alias/${keyAlias}`, description: `Default bucket encryption key`, }); encryptionKey.addToResourcePolicy( @@ -21,7 +27,10 @@ export function createDefaultS3Key(props: { accountStack: AccountStack }): kms.K resources: ['*'], }), ); - return encryptionKey; + return { + encryptionKey, + alias: keyAlias, + }; } /** diff --git a/src/deployments/cdk/src/deployments/defaults/step-1.ts b/src/deployments/cdk/src/deployments/defaults/step-1.ts index c6060f73d..98c5a4519 100644 --- a/src/deployments/cdk/src/deployments/defaults/step-1.ts +++ b/src/deployments/cdk/src/deployments/defaults/step-1.ts @@ -11,7 +11,7 @@ import { createEncryptionKeyName, createRoleName, } from '@aws-accelerator/cdk-accelerator/src/core/accelerator-name-generator'; -import { CfnLogBucketOutput, CfnAesBucketOutput, CfnCentralBucketOutput } from './outputs'; +import { CfnLogBucketOutput, CfnAesBucketOutput, CfnCentralBucketOutput, CfnEbsKmsOutput } from './outputs'; import { AccountStacks } from '../../common/account-stacks'; import { Account } from '../../utils/accounts'; import { createDefaultS3Bucket, createDefaultS3Key } from './shared'; @@ -84,8 +84,9 @@ function createCentralBucketCopy(props: DefaultsStep1Props) { bucketName: centralBucketName, }); + const keyAlias = createEncryptionKeyName('Config-Key'); const encryptionKey = new kms.Key(masterAccountStack, 'CentralBucketKey', { - alias: 'alias/' + createEncryptionKeyName('Config-Key'), + alias: `alias/${keyAlias}`, description: 'Key used to encrypt/decrypt the copy of central S3 bucket', }); @@ -137,6 +138,8 @@ function createCentralBucketCopy(props: DefaultsStep1Props) { bucketName: bucket.bucketName, encryptionKeyArn: encryptionKey.keyArn, region: cdk.Aws.REGION, + encryptionKeyId: encryptionKey.keyId, + encryptionKeyName: keyAlias, }); return bucket; @@ -162,7 +165,7 @@ function createCentralLogBucket(props: DefaultsStep1Props) { const logBucket = createDefaultS3Bucket({ accountStack: logAccountStack, - encryptionKey: logKey, + encryptionKey: logKey.encryptionKey, logRetention: defaultLogRetention!, }); @@ -265,6 +268,8 @@ function createCentralLogBucket(props: DefaultsStep1Props) { bucketName: logBucket.bucketName, encryptionKeyArn: logBucket.encryptionKey!.keyArn, region: cdk.Aws.REGION, + encryptionKeyId: logBucket.encryptionKey!.keyId, + encryptionKeyName: logKey.alias, }); logBucket.encryptionKey?.addToResourcePolicy( @@ -362,9 +367,10 @@ function createDefaultEbsEncryptionKey(props: DefaultsStep1Props): AccountRegion continue; } + const keyAlias = createEncryptionKeyName('EBS-Key'); // Default EBS encryption key const key = new kms.Key(accountStack, 'EbsDefaultEncryptionKey', { - alias: 'alias/' + createEncryptionKeyName('EBS-Key'), + alias: `alias/${keyAlias}`, description: 'Key used to encrypt/decrypt EBS by default', }); @@ -386,6 +392,12 @@ function createDefaultEbsEncryptionKey(props: DefaultsStep1Props): AccountRegion ...accountEbsEncryptionKeys[localAccountKey], [region]: key, }; + + new CfnEbsKmsOutput(accountStack, 'EbsEncryptionKey', { + encryptionKeyName: keyAlias, + encryptionKeyId: key.keyId, + encryptionKeyArn: key.keyArn, + }); } } return accountEbsEncryptionKeys; diff --git a/src/deployments/cdk/src/deployments/defaults/step-2.ts b/src/deployments/cdk/src/deployments/defaults/step-2.ts index 159d46cc1..b0c0d9716 100644 --- a/src/deployments/cdk/src/deployments/defaults/step-2.ts +++ b/src/deployments/cdk/src/deployments/defaults/step-2.ts @@ -55,7 +55,7 @@ function createDefaultS3Buckets(props: DefaultsStep2Props) { const bucket = createDefaultS3Bucket({ accountStack, - encryptionKey: key, + encryptionKey: key.encryptionKey, logRetention, }); @@ -95,6 +95,8 @@ function createDefaultS3Buckets(props: DefaultsStep2Props) { bucketName: bucket.bucketName, encryptionKeyArn: bucket.encryptionKey!.keyArn, region: cdk.Aws.REGION, + encryptionKeyId: bucket.encryptionKey!.keyId, + encryptionKeyName: key.alias, }); } diff --git a/src/deployments/cdk/src/deployments/secrets/outputs.ts b/src/deployments/cdk/src/deployments/secrets/outputs.ts index b4f3a45d3..262313854 100644 --- a/src/deployments/cdk/src/deployments/secrets/outputs.ts +++ b/src/deployments/cdk/src/deployments/secrets/outputs.ts @@ -2,6 +2,8 @@ import * as t from 'io-ts'; export const SecretEncryptionKeyOutputType = t.interface( { + encryptionKeyName: t.string, + encryptionKeyId: t.string, encryptionKeyArn: t.string, }, 'SecretEncryptionKeyOutput', diff --git a/src/deployments/cdk/src/deployments/secrets/step-1.ts b/src/deployments/cdk/src/deployments/secrets/step-1.ts index fa9938e29..8e627313f 100644 --- a/src/deployments/cdk/src/deployments/secrets/step-1.ts +++ b/src/deployments/cdk/src/deployments/secrets/step-1.ts @@ -21,6 +21,8 @@ export async function step1(props: SecretsStep1Props) { new StructuredOutput(masterAccountStack, 'SecretEncryptionKey', { type: SecretEncryptionKeyOutputType, value: { + encryptionKeyName: secretsContainer.alias, + encryptionKeyId: secretsContainer.encryptionKey.keyId, encryptionKeyArn: secretsContainer.encryptionKey.keyArn, }, }); diff --git a/src/deployments/cdk/src/deployments/ssm/session-manager.ts b/src/deployments/cdk/src/deployments/ssm/session-manager.ts index e96e6b1b5..9c880cc98 100644 --- a/src/deployments/cdk/src/deployments/ssm/session-manager.ts +++ b/src/deployments/cdk/src/deployments/ssm/session-manager.ts @@ -13,7 +13,7 @@ import { getVpcSharedAccountKeys } from '../../common/vpc-subnet-sharing'; import { Account } from '../../utils/accounts'; import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; import { SSMSessionManagerDocument } from '@aws-accelerator/custom-resource-ssm-session-manager-document'; -import { AccountBuckets } from '../defaults'; +import { AccountBuckets, CfnSsmKmsOutput } from '../defaults'; export interface SSMStep1Props { accountStacks: AccountStacks; @@ -67,8 +67,9 @@ export async function step1(props: SSMStep1Props) { continue; } + const keyAlias = createEncryptionKeyName('SSM-Key'); const ssmKey = new Key(accountStack, 'SSM-Key', { - alias: 'alias/' + createEncryptionKeyName('SSM-Key'), + alias: `alias/${keyAlias}`, trustAccountIdentities: true, }); ssmKey.grantEncryptDecrypt(new AccountPrincipal(cdk.Aws.ACCOUNT_ID)); @@ -98,6 +99,12 @@ export async function step1(props: SSMStep1Props) { ...accountRegionSsmDocuments[localAccountKey], [region]: ssmKey, }; + + new CfnSsmKmsOutput(accountStack, 'SsmEncryptionKey', { + encryptionKeyName: keyAlias, + encryptionKeyId: ssmKey.keyId, + encryptionKeyArn: ssmKey.keyArn, + }); } } } diff --git a/src/lib/cdk-accelerator/src/core/secrets-container.ts b/src/lib/cdk-accelerator/src/core/secrets-container.ts index d94604fee..3f5a19c78 100644 --- a/src/lib/cdk-accelerator/src/core/secrets-container.ts +++ b/src/lib/cdk-accelerator/src/core/secrets-container.ts @@ -30,13 +30,15 @@ export interface SecretsContainerProps extends Omit; + +export const EbsKmsOutputFinder = createStructuredOutputFinder(EbsKmsOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); diff --git a/src/lib/common-outputs/src/ssm.ts b/src/lib/common-outputs/src/ssm.ts index f9d37def4..b4ec025f0 100644 --- a/src/lib/common-outputs/src/ssm.ts +++ b/src/lib/common-outputs/src/ssm.ts @@ -21,3 +21,23 @@ export const IamRoleOutputFinder = createStructuredOutputFinder(SSMOutput, finde predicate: o => o.roleKey === props.roleKey, }), })); + +export const SsmKmsOutput = t.interface( + { + encryptionKeyName: t.string, + encryptionKeyId: t.string, + encryptionKeyArn: t.string, + }, + 'SsmKms', +); + +export type SsmKmsOutput = t.TypeOf; + +export const SsmKmsOutputFinder = createStructuredOutputFinder(SsmKmsOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); From 6709c88ac921c8fc2e3c449968e3b7bca893aaf9 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 23 Sep 2020 17:26:04 +0530 Subject: [PATCH 15/25] Saving ELB Outputs --- .../src/save-outputs-to-ssm/elb-outputs.ts | 298 ++++++++++++++++++ .../src/save-outputs-to-ssm/iam-outputs.ts | 6 +- .../src/save-outputs-to-ssm/iam-utils.ts | 8 +- .../runtime/src/save-outputs-to-ssm/index.ts | 18 +- .../runtime/src/save-outputs-to-ssm/utils.ts | 2 +- .../cdk/src/deployments/alb/outputs.ts | 5 + .../cdk/src/deployments/alb/step-1.ts | 9 + .../src/deployments/certificates/outputs.ts | 2 +- .../cdk/src/deployments/rsyslog/step-2.ts | 9 + src/lib/cdk-constructs/src/vpc/alb.ts | 12 + src/lib/cdk-constructs/src/vpc/nlb.ts | 4 + src/lib/common-config/src/index.ts | 1 + src/lib/common-outputs/src/elb.ts | 22 ++ src/lib/common-outputs/src/iam-role.ts | 2 +- 14 files changed, 384 insertions(+), 14 deletions(-) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/elb-outputs.ts create mode 100644 src/deployments/cdk/src/deployments/alb/outputs.ts create mode 100644 src/lib/common-outputs/src/elb.ts diff --git a/src/core/runtime/src/save-outputs-to-ssm/elb-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/elb-outputs.ts new file mode 100644 index 000000000..db689b94e --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/elb-outputs.ts @@ -0,0 +1,298 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, getIndexOutput, saveIndexOutput } from './utils'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { LoadBalancerOutputFinder, LoadBalancerOutput } from '@aws-accelerator/common-outputs/src/elb'; + +interface OutputUtilLbType extends OutputUtilGenericType { + account: string; +} +interface OutputUtilElb { + albs?: OutputUtilLbType[]; + nlbs?: OutputUtilLbType[]; +} + +/** + * Outputs for elb related deployments will be found in following phases + * - Phase-3 + */ + +/** + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveElbOutputs(props: SaveOutputsInput) { + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; + const oldElbOutputUtils = await getIndexOutput(outputUtilsTableName, `${account.key}-${region}-lelb`, dynamodb); + // Existing index check happens on this variable + let elbOutputUtils: OutputUtilElb; + if (oldElbOutputUtils) { + elbOutputUtils = oldElbOutputUtils; + } else { + elbOutputUtils = { + albs: [], + nlbs: [], + }; + } + + // Storing new resource index and updating DDB in this variable + const newElbOutputs: OutputUtilElb = { + albs: [], + nlbs: [], + }; + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + + const localOutputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-3`, dynamodb); + const elbOutputs = LoadBalancerOutputFinder.findAll({ + outputs: localOutputs, + accountKey: account.key, + region, + }); + const nlbOutputs = elbOutputs.filter(elb => elb.type === 'NETWORK'); + const albOutputs = elbOutputs.filter(elb => elb.type === 'APPLICATION'); + newElbOutputs.nlbs = ( + await saveElbOutputsImpl({ + acceleratorPrefix, + lbOutputs: nlbOutputs, + lbUtil: elbOutputUtils.nlbs || [], + ssm, + type: 'nlb', + accountKey: account.key, + source: 'local', + }) + ).lbs; + + newElbOutputs.albs = ( + await saveElbOutputsImpl({ + acceleratorPrefix, + lbOutputs: albOutputs, + lbUtil: elbOutputUtils.albs || [], + ssm, + type: 'alb', + accountKey: account.key, + source: 'local', + }) + ).lbs; + + await saveIndexOutput(outputUtilsTableName, `${account.key}-${region}-lelb`, JSON.stringify(newElbOutputs), dynamodb); + + const accountConfigAndKey = config.getAccountConfigs().find(([accountKey, _]) => accountKey === account.key); + if (!accountConfigAndKey) { + return; + } + const accountConfig = accountConfigAndKey[1]; + if (!accountConfig['populate-all-elbs-in-param-store']) { + return; + } + const additionalNlbAccountKeys = Array.from( + new Set([ + ...config + .getVpcConfigs() + .filter(c => c.deployments?.rsyslog && c.accountKey !== account.key) + .map(al => al.accountKey), + ]), + ); + const additionalAlbAccountKeys = config + .getAlbConfigs() + .filter(lb => lb.accountKey !== account.key) + .map(al => al.accountKey); + const additionalAccountKeys = Array.from(new Set([...additionalNlbAccountKeys, ...additionalAlbAccountKeys])); + + const remoteOldElbOutputUtils = await getIndexOutput(outputUtilsTableName, `${account.key}-${region}-elb`, dynamodb); + // Existing index check happens on this variable + let remoteElbOutputUtils: OutputUtilElb; + if (remoteOldElbOutputUtils) { + remoteElbOutputUtils = remoteOldElbOutputUtils; + } else { + remoteElbOutputUtils = {}; + } + if (!remoteElbOutputUtils.albs) { + remoteElbOutputUtils.albs = []; + } + if (!remoteElbOutputUtils.nlbs) { + remoteElbOutputUtils.nlbs = []; + } + + const previousNlbAccountKeys = Array.from(new Set([...(remoteElbOutputUtils.nlbs.flatMap(al => al.account) || [])])); + const previousAlbAccountKeys = Array.from(new Set([...(remoteElbOutputUtils.albs.flatMap(al => al.account) || [])])); + const previousElbAccountKeys = Array.from(new Set([...previousNlbAccountKeys, ...previousAlbAccountKeys])); + + if (additionalAccountKeys.length === 0 && previousElbAccountKeys.length === 0) { + return; + } + + // Storing new resource index and updating DDB in this variable + const newRemoteElbOutputs: OutputUtilElb = {}; + newRemoteElbOutputs.albs = []; + newRemoteElbOutputs.nlbs = []; + + const nlbIndices = remoteElbOutputUtils.nlbs.flatMap(lb => lb.index) || []; + let maxNlbIndex = nlbIndices.length === 0 ? 0 : Math.max(...nlbIndices); + + const albIndices = remoteElbOutputUtils.albs.flatMap(lb => lb.index) || []; + let maxAlbIndex = albIndices.length === 0 ? 0 : Math.max(...albIndices); + + for (const accountKey of additionalAccountKeys) { + const remoteOutputs: StackOutput[] = await getOutput(outputsTableName, `${accountKey}-${region}-3`, dynamodb); + const remoteElbOutputs = LoadBalancerOutputFinder.findAll({ + outputs: remoteOutputs, + accountKey, + region, + }); + + const remoteNlbOutputs = remoteElbOutputs.filter(elb => elb.type === 'NETWORK'); + const remoteAlbOutputs = remoteElbOutputs.filter(elb => elb.type === 'APPLICATION'); + + if (remoteElbOutputs.length === 0 && remoteNlbOutputs.length === 0 && remoteAlbOutputs.length === 0) { + continue; + } + const saveNlbOp = await saveElbOutputsImpl({ + acceleratorPrefix, + lbOutputs: remoteNlbOutputs, + lbUtil: remoteElbOutputUtils.nlbs?.filter(lb => lb.account === accountKey) || [], + ssm, + type: 'nlb', + accountKey, + source: 'remote', + maxIndex: maxNlbIndex, + }); + newRemoteElbOutputs.nlbs.push(...saveNlbOp.lbs); + maxNlbIndex = saveNlbOp.currentMaxIndex!; + + const saveAlbOp = await saveElbOutputsImpl({ + acceleratorPrefix, + lbOutputs: remoteAlbOutputs, + lbUtil: remoteElbOutputUtils.albs?.filter(lb => lb.account === accountKey) || [], + ssm, + type: 'alb', + accountKey, + source: 'remote', + maxIndex: maxAlbIndex, + }); + newRemoteElbOutputs.albs.push(...saveAlbOp.lbs); + maxAlbIndex = saveAlbOp.currentMaxIndex!; + } + + await saveIndexOutput( + outputUtilsTableName, + `${account.key}-${region}-elb`, + JSON.stringify(newRemoteElbOutputs), + dynamodb, + ); + + const removeNlbAccounts = previousNlbAccountKeys.filter(lb => !additionalNlbAccountKeys.includes(lb)); + const removeAlbAccounts = previousAlbAccountKeys.filter(lb => !additionalAlbAccountKeys.includes(lb)); + const removalNlbs = removeNlbAccounts + .map(lb => + (remoteElbOutputUtils.nlbs || []) + .filter(l => l.account === lb) + .map(nlb => [ + `/${acceleratorPrefix}/elb/nlb/${nlb.index}/name`, + `/${acceleratorPrefix}/elb/nlb/${nlb.index}/dns`, + `/${acceleratorPrefix}/elb/nlb/${nlb.index}/account`, + ]), + ) + .flatMap(s => s) + .flatMap(r => r); + const removalAlbs = removeAlbAccounts + .map(lb => + (remoteElbOutputUtils.albs || []) + .filter(l => l.account === lb) + .map(alb => [ + `/${acceleratorPrefix}/elb/alb/${alb.index}/name`, + `/${acceleratorPrefix}/elb/alb/${alb.index}/dns`, + `/${acceleratorPrefix}/elb/alb/${alb.index}/account`, + ]), + ) + .flatMap(s => s) + .flatMap(r => r); + const removeNames: string[] = [...removalNlbs, ...removalAlbs]; + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } +} + +async function saveElbOutputsImpl(props: { + lbOutputs: LoadBalancerOutput[]; + ssm: SSM; + acceleratorPrefix: string; + lbUtil: OutputUtilLbType[]; + type: 'alb' | 'nlb'; + accountKey: string; + source: 'local' | 'remote'; + maxIndex?: number; +}): Promise<{ + lbs: OutputUtilLbType[]; + currentMaxIndex?: number; +}> { + const { acceleratorPrefix, lbOutputs, lbUtil, ssm, type, accountKey, source } = props; + const lbPrefix = source === 'local' ? 'lelb' : 'elb'; + if (lbUtil.length === 0 && lbOutputs.length === 0) { + return { + lbs: [], + }; + } + const newLbUtils: OutputUtilLbType[] = []; + let maxIndex: number; + if (props.maxIndex) { + maxIndex = props.maxIndex; + } else { + const indices = lbUtil.flatMap(lb => lb.index) || []; + maxIndex = indices.length === 0 ? 0 : Math.max(...indices); + } + for (const nlbOutput of lbOutputs) { + let currentIndex: number; + const previousIndex = lbUtil.findIndex(lb => lb.name === nlbOutput.name && lb.account === accountKey); + if (previousIndex >= 0) { + currentIndex = lbUtil[previousIndex].index; + } else { + currentIndex = ++maxIndex; + } + newLbUtils.push({ + name: nlbOutput.name, + index: currentIndex, + account: accountKey, + }); + if (previousIndex < 0) { + await ssm.putParameter(`/${acceleratorPrefix}/${lbPrefix}/${type}/${currentIndex}/name`, nlbOutput.displayName); + await ssm.putParameter(`/${acceleratorPrefix}/${lbPrefix}/${type}/${currentIndex}/dns`, nlbOutput.dnsName); + if (source === 'remote') { + await ssm.putParameter(`/${acceleratorPrefix}/${lbPrefix}/${type}/${currentIndex}/account`, accountKey); + } + } else { + lbUtil.splice(previousIndex, 1); + } + } + + const removeNames = lbUtil + .map(lb => [ + `/${acceleratorPrefix}/${lbPrefix}/${type}/${lb.index}/name`, + `/${acceleratorPrefix}/${lbPrefix}/${type}/${lb.index}/dns`, + `/${acceleratorPrefix}/${lbPrefix}/${type}/${lb.index}/account`, + ]) + .flatMap(s => s); + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } + return { + lbs: newLbUtils, + currentMaxIndex: maxIndex, + }; +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts index 19da0668a..18ad0aaf7 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-outputs.ts @@ -49,10 +49,10 @@ export async function saveIamOutputs(props: SaveOutputsInput) { const accountsIam: { [accountKey: string]: IamConfig[] } = {}; // finding iam for account iam configs - prepareMandatoryAccountIamConfigs(accountsIam, accountConfig); + await prepareMandatoryAccountIamConfigs(accountsIam, accountConfig); // finding iam for ou iam configs - prepareOuAccountIamConfigs(config, account, accountsIam); + await prepareOuAccountIamConfigs(config, account, accountsIam); // console.log('Accounts Iam', Object.keys(accountsIam)); // if no IAM Config found for the account, return @@ -123,5 +123,5 @@ export async function saveIamOutputs(props: SaveOutputsInput) { }; const iamIndexOutput = JSON.stringify(iamOutput); console.log('indexOutput', iamIndexOutput); - saveIndexOutput(outputUtilsTableName, `${account.key}-${region}-identity`, iamIndexOutput, dynamodb); + await saveIndexOutput(outputUtilsTableName, `${account.key}-${region}-identity`, iamIndexOutput, dynamodb); } diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts index f766f59ba..52e9a186e 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -101,7 +101,7 @@ export async function saveIamUsers( }); const removalIndex = removalObjects.findIndex(p => p.name === userOutput.userName); - if (removalIndex != -1) { + if (removalIndex !== -1) { removalObjects.splice(removalIndex, 1); } } @@ -173,7 +173,7 @@ export async function saveIamGroups( }); const removalIndex = removalObjects.findIndex(p => p.name === groupOutput.groupName); - if (removalIndex != -1) { + if (removalIndex !== -1) { removalObjects.splice(removalIndex, 1); } } @@ -244,7 +244,7 @@ export async function saveIamPolicy( }); const removalIndex = removalObjects.findIndex(p => p.name === policyOutput.policyName); - if (removalIndex != -1) { + if (removalIndex !== -1) { removalObjects.splice(removalIndex, 1); } } @@ -316,7 +316,7 @@ export async function saveIamRoles( }); const removalIndex = removalObjects.findIndex(p => p.name === roleOutput.roleName); - if (removalIndex != -1) { + if (removalIndex !== -1) { removalObjects.splice(removalIndex, 1); } } diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 97f34cf9e..e2440d5aa 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -1,11 +1,10 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { saveNetworkOutputs } from './network-outputs'; -import { SSM } from '@aws-accelerator/common/src/aws/ssm'; import { saveIamOutputs } from './iam-outputs'; +import { saveElbOutputs } from './elb-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -17,10 +16,9 @@ export interface SaveOutputsToSsmInput extends LoadConfigurationInput { } const dynamodb = new DynamoDB(); -const sts = new STS(); export const handler = async (input: SaveOutputsToSsmInput) => { - console.log(`Adding service control policy to organization...`); + console.log(`Saving SM Outputs to SSM Parameter store...`); console.log(JSON.stringify(input, null, 2)); const { @@ -77,6 +75,18 @@ export const handler = async (input: SaveOutputsToSsmInput) => { outputUtilsTableName, }); + // Store ELB Outputs to SSM Parameter Store + await saveElbOutputs({ + acceleratorPrefix, + account, + assumeRoleName, + config, + dynamodb, + outputUtilsTableName, + outputsTableName, + region, + }); + return { status: 'SUCCESS', }; diff --git a/src/core/runtime/src/save-outputs-to-ssm/utils.ts b/src/core/runtime/src/save-outputs-to-ssm/utils.ts index 539ceea48..be4bd6a8a 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/utils.ts @@ -59,7 +59,7 @@ export async function saveIndexOutput( key: 'v', name: 'value', type: 'S', - value: value, + value, }, ]); await dynamodb.updateItem({ diff --git a/src/deployments/cdk/src/deployments/alb/outputs.ts b/src/deployments/cdk/src/deployments/alb/outputs.ts new file mode 100644 index 000000000..0d65df3fb --- /dev/null +++ b/src/deployments/cdk/src/deployments/alb/outputs.ts @@ -0,0 +1,5 @@ +import { LoadBalancerOutput } from '@aws-accelerator/common-outputs/src/elb'; +import { StaticResourcesOutput } from '@aws-accelerator/common-outputs/src/static-resource'; +import { createCfnStructuredOutput } from '../../common/structured-output'; +export const CfnLoadBalancerOutput = createCfnStructuredOutput(LoadBalancerOutput); +export const CfnStaticResourcesOutput = createCfnStructuredOutput(StaticResourcesOutput); diff --git a/src/deployments/cdk/src/deployments/alb/step-1.ts b/src/deployments/cdk/src/deployments/alb/step-1.ts index 2dfd620bc..557233adc 100644 --- a/src/deployments/cdk/src/deployments/alb/step-1.ts +++ b/src/deployments/cdk/src/deployments/alb/step-1.ts @@ -6,6 +6,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as s3 from '@aws-cdk/aws-s3'; import { ApplicationLoadBalancer } from '@aws-accelerator/cdk-constructs/src/vpc'; +import { CfnLoadBalancerOutput } from './outputs'; import { AcceleratorConfig, AlbConfig, @@ -173,6 +174,14 @@ export function createAlb( actionType: albConfig['action-type'], targetGroupArns: targetGroupIds, }); + + new CfnLoadBalancerOutput(accountStack, `Alb${albConfig.name}-Output`, { + displayName: balancer.name, + dnsName: balancer.dns, + hostedZoneId: balancer.hostedZoneId, + name: albConfig.name, + type: 'APPLICATION', + }); } export function getTargetGroupArn(props: { diff --git a/src/deployments/cdk/src/deployments/certificates/outputs.ts b/src/deployments/cdk/src/deployments/certificates/outputs.ts index 255c678e2..0734272a7 100644 --- a/src/deployments/cdk/src/deployments/certificates/outputs.ts +++ b/src/deployments/cdk/src/deployments/certificates/outputs.ts @@ -26,4 +26,4 @@ export const AcmOutputFinder = createStructuredOutputFinder(AcmOutput, finder => accountKey: props.accountKey, region: props.region, }), -})); \ No newline at end of file +})); diff --git a/src/deployments/cdk/src/deployments/rsyslog/step-2.ts b/src/deployments/cdk/src/deployments/rsyslog/step-2.ts index 66957e6a4..42da48bd9 100644 --- a/src/deployments/cdk/src/deployments/rsyslog/step-2.ts +++ b/src/deployments/cdk/src/deployments/rsyslog/step-2.ts @@ -16,6 +16,7 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { ImageIdOutputFinder } from '@aws-accelerator/common-outputs/src/ami-output'; import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; import { Context } from '../../utils/context'; +import { CfnLoadBalancerOutput } from '../alb/outputs'; export interface RSysLogStep1Props { accountStacks: AccountStacks; @@ -108,6 +109,14 @@ export function createNlb( name: balancer.name, dns: balancer.dns, }); + + new CfnLoadBalancerOutput(accountStack, `NlbRsyslog${accountKey}-Output`, { + displayName: balancer.name, + dnsName: balancer.dns, + hostedZoneId: balancer.hostedZoneId, + name: 'RsyslogNLB', + type: 'NETWORK', + }); } export function createAsg( diff --git a/src/lib/cdk-constructs/src/vpc/alb.ts b/src/lib/cdk-constructs/src/vpc/alb.ts index 0d20da8d7..7f19ffb6b 100644 --- a/src/lib/cdk-constructs/src/vpc/alb.ts +++ b/src/lib/cdk-constructs/src/vpc/alb.ts @@ -79,4 +79,16 @@ export class ApplicationLoadBalancer extends cdk.Construct { }); this.listeners.push(listener); } + + get name(): string { + return this.resource.name!; + } + + get dns(): string { + return this.resource.attrDnsName; + } + + get hostedZoneId(): string { + return this.resource.attrCanonicalHostedZoneId; + } } diff --git a/src/lib/cdk-constructs/src/vpc/nlb.ts b/src/lib/cdk-constructs/src/vpc/nlb.ts index db9d4901b..7195abf38 100644 --- a/src/lib/cdk-constructs/src/vpc/nlb.ts +++ b/src/lib/cdk-constructs/src/vpc/nlb.ts @@ -54,4 +54,8 @@ export class NetworkLoadBalancer extends cdk.Construct { get dns(): string { return this.resource.attrDnsName; } + + get hostedZoneId(): string { + return this.resource.attrCanonicalHostedZoneId; + } } diff --git a/src/lib/common-config/src/index.ts b/src/lib/common-config/src/index.ts index ddaa1b39c..8229b1a70 100644 --- a/src/lib/common-config/src/index.ts +++ b/src/lib/common-config/src/index.ts @@ -554,6 +554,7 @@ export const MandatoryAccountConfigType = t.interface({ 'src-filename': t.string, 'exclude-ou-albs': optional(t.boolean), 'keep-default-vpc-regions': fromNullable(t.array(t.string), []), + 'populate-all-elbs-in-param-store': fromNullable(t.boolean, false), }); export type MandatoryAccountConfig = t.TypeOf; diff --git a/src/lib/common-outputs/src/elb.ts b/src/lib/common-outputs/src/elb.ts new file mode 100644 index 000000000..949bf7bc3 --- /dev/null +++ b/src/lib/common-outputs/src/elb.ts @@ -0,0 +1,22 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; +import { enumType } from '@aws-accelerator/common-types'; + +export const LOADBALANCER = ['APPLICATION', 'NETWORK'] as const; + +export const LoadBalancerType = enumType(LOADBALANCER, 'LoadBalancerType'); + +export const LoadBalancerOutput = t.interface( + { + hostedZoneId: t.string, + dnsName: t.string, + name: t.string, + displayName: t.string, + type: LoadBalancerType, + }, + 'LoadBalancerOutput', +); + +export type LoadBalancerOutput = t.TypeOf; + +export const LoadBalancerOutputFinder = createStructuredOutputFinder(LoadBalancerOutput, () => ({})); diff --git a/src/lib/common-outputs/src/iam-role.ts b/src/lib/common-outputs/src/iam-role.ts index a302d0db2..0cb2817f4 100644 --- a/src/lib/common-outputs/src/iam-role.ts +++ b/src/lib/common-outputs/src/iam-role.ts @@ -47,6 +47,6 @@ export const IamPolicyOutputFinder = createStructuredOutputFinder(IamPolicyOutpu finder.tryFindOne({ outputs: props.outputs, accountKey: props.accountKey, - predicate: o => o.policyKey === props.policyKey && o.policyName == props.policyName, + predicate: o => o.policyKey === props.policyKey && o.policyName === props.policyName, }), })); From 323627559c3f7fcc8dede99d8bf3d2576ad99248 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 23 Sep 2020 19:40:58 +0530 Subject: [PATCH 16/25] Save Event outputs --- .../src/save-outputs-to-ssm/event-outputs.ts | 111 ++++++++++++++++++ .../runtime/src/save-outputs-to-ssm/index.ts | 13 ++ .../cdk/src/deployments/sns/step-1.ts | 7 ++ 3 files changed, 131 insertions(+) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/event-outputs.ts diff --git a/src/core/runtime/src/save-outputs-to-ssm/event-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/event-outputs.ts new file mode 100644 index 000000000..bcb912ba8 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/event-outputs.ts @@ -0,0 +1,111 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, getIndexOutput, saveIndexOutput } from './utils'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { SnsTopicOutputFinder } from '@aws-accelerator/common-outputs/src/sns-topic'; + +interface OutputUtilEvent { + events?: OutputUtilGenericType[]; +} + +/** + * Outputs for event related deployments will be found in following phases + * - Phase-2 + */ + +/** + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveEventOutputs(props: SaveOutputsInput) { + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; + const logAccountKey = config.getMandatoryAccountKey('central-log'); + if (account.key !== logAccountKey) { + console.info('Ignoring storing event ouputs since we only need them in log account'); + return; + } + + if (config['global-options']['central-log-services']['sns-excl-regions']?.includes(region)) { + console.info('Ignoring storing event outputs since region is in exclusion list'); + return; + } + const oldEventOutputUtils = await getIndexOutput(outputUtilsTableName, `${account.key}-${region}-event`, dynamodb); + // Existing index check happens on this variable + let eventOutputUtils: OutputUtilEvent; + if (oldEventOutputUtils) { + eventOutputUtils = oldEventOutputUtils; + } else { + eventOutputUtils = {}; + } + if (!eventOutputUtils.events) { + eventOutputUtils.events = []; + } + + // Storing new resource index and updating DDB in this variable + const newEventOutputs: OutputUtilEvent = {}; + newEventOutputs.events = []; + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + + const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-2`, dynamodb); + const eventOutputs = SnsTopicOutputFinder.findAll({ + outputs, + accountKey: logAccountKey, + region, + }); + + const indices = eventOutputUtils.events.flatMap(e => e.index) || []; + let maxIndex = indices.length === 0 ? 0 : Math.max(...indices); + + for (const eventOutput of eventOutputs) { + let currentIndex: number; + const previousIndex = eventOutputUtils.events.findIndex(e => e.name === eventOutput.topicName); + if (previousIndex >= 0) { + currentIndex = eventOutputUtils.events[previousIndex].index; + } else { + currentIndex = ++maxIndex; + } + newEventOutputs.events.push({ + index: currentIndex, + name: eventOutput.topicName, + }); + + if (previousIndex < 0) { + await ssm.putParameter(`/${acceleratorPrefix}/event/${currentIndex}/name`, `${eventOutput.topicName}`); + await ssm.putParameter(`/${acceleratorPrefix}/event/${currentIndex}/arn`, `${eventOutput.topicArn}`); + } + + if (previousIndex >= 0) { + eventOutputUtils.events.splice(previousIndex, 1); + } + } + + await saveIndexOutput( + outputUtilsTableName, + `${account.key}-${region}-event`, + JSON.stringify(newEventOutputs), + dynamodb, + ); + const removeNames = eventOutputUtils.events + .map(e => [`/${acceleratorPrefix}/event/${e.index}/name`, `/${acceleratorPrefix}/event/${e.index}/arn`]) + .flatMap(es => es); + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index e2440d5aa..4354bb770 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -5,6 +5,7 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { saveNetworkOutputs } from './network-outputs'; import { saveIamOutputs } from './iam-outputs'; import { saveElbOutputs } from './elb-outputs'; +import { saveEventOutputs } from './event-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -87,6 +88,18 @@ export const handler = async (input: SaveOutputsToSsmInput) => { region, }); + // Store Event Outputs to SSM Parameter Store + await saveEventOutputs({ + acceleratorPrefix, + account, + assumeRoleName, + config, + dynamodb, + outputUtilsTableName, + outputsTableName, + region, + }); + return { status: 'SUCCESS', }; diff --git a/src/deployments/cdk/src/deployments/sns/step-1.ts b/src/deployments/cdk/src/deployments/sns/step-1.ts index 37fb69ffd..c36c25f7b 100644 --- a/src/deployments/cdk/src/deployments/sns/step-1.ts +++ b/src/deployments/cdk/src/deployments/sns/step-1.ts @@ -9,6 +9,7 @@ import * as cdk from '@aws-cdk/core'; import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import * as iam from '@aws-cdk/aws-iam'; +import { CfnSnsTopicOutput } from './outputs'; export interface SnsStep1Props { accountStacks: AccountStacks; @@ -121,6 +122,12 @@ export async function step1(props: SnsStep1Props) { endpoint: snsSubscriberFunc.functionArn, }); } + + new CfnSnsTopicOutput(accountStack, `SnsNotificationTopic${notificationType}-Otuput`, { + topicArn: topic.topicArn, + topicKey: notificationType, + topicName: topic.topicName, + }); } } } From 37f42e23ae4f2e6e9d658cc9e8d3044e431cbb4f Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 23 Sep 2020 20:36:16 +0530 Subject: [PATCH 17/25] added encrypt outputs to parameter store --- .../save-outputs-to-ssm/encrypt-outputs.ts | 66 +++++ .../src/save-outputs-to-ssm/encrypt-utils.ts | 246 ++++++++++++++++++ .../runtime/src/save-outputs-to-ssm/index.ts | 12 + .../src/deployments/certificates/outputs.ts | 23 +- .../cdk/src/deployments/defaults/outputs.ts | 28 -- src/lib/common-outputs/src/buckets.ts | 72 +++++ src/lib/common-outputs/src/certificates.ts | 22 ++ src/lib/common-outputs/src/secrets.ts | 16 ++ 8 files changed, 435 insertions(+), 50 deletions(-) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/encrypt-outputs.ts create mode 100644 src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts create mode 100644 src/lib/common-outputs/src/buckets.ts create mode 100644 src/lib/common-outputs/src/certificates.ts diff --git a/src/core/runtime/src/save-outputs-to-ssm/encrypt-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/encrypt-outputs.ts new file mode 100644 index 000000000..9f50a800f --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/encrypt-outputs.ts @@ -0,0 +1,66 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, saveIndexOutput, getIamSsmOutput } from './utils'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { saveKmsKeys, saveAcm } from './encrypt-utils'; + +interface EncryptOutput { + kms: OutputUtilGenericType[]; + acm: OutputUtilGenericType[]; +} + +/** + * + * Outputs for kms and ACM related deployments will be found in following phases + * - Phase-0 + * - Phase-1 + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveEncryptsOutputs(props: SaveOutputsInput) { + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; + + const smRegion = config['global-options']['aws-org-master'].region; + const phase0Outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${smRegion}-0`, dynamodb); + const phase1Outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${smRegion}-1`, dynamodb); + const outputs = [...phase0Outputs, ...phase1Outputs]; + const encryptOutputs = await getIamSsmOutput(outputUtilsTableName, `${account.key}-${region}-encrypt`, dynamodb); + + const kms: OutputUtilGenericType[] = []; + const acm: OutputUtilGenericType[] = []; + + if (encryptOutputs) { + const ssmIamOutput: EncryptOutput = JSON.parse(encryptOutputs); + kms.push(...ssmIamOutput.kms); + acm.push(...ssmIamOutput.acm); + } + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + + const updatedKms = await saveKmsKeys(config, outputs, ssm, account, region, acceleratorPrefix, kms); + const updatedAcm = await saveAcm(config, outputs, ssm, account, region, acceleratorPrefix, acm); + + const encryptOutput: EncryptOutput = { + kms: updatedKms, + acm: updatedAcm, + }; + const encryptIndexOutput = JSON.stringify(encryptOutput); + console.log('encryptIndexOutput', encryptIndexOutput); + await saveIndexOutput(outputUtilsTableName, `${account.key}-${region}-encrypt`, encryptIndexOutput, dynamodb); +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts new file mode 100644 index 000000000..eb4180c8d --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts @@ -0,0 +1,246 @@ +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { AcceleratorConfig } from '@aws-accelerator/common-config'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { OutputUtilGenericType } from './utils'; +import { EbsKmsOutputFinder } from '@aws-accelerator/common-outputs/src/ebs'; +import { + AccountBucketOutputFinder, + LogBucketOutputTypeOutputFinder, +} from '@aws-accelerator/common-outputs/src/buckets'; +import { CentralBucketOutputFinder } from '@aws-accelerator/common-outputs/src/central-bucket'; +import { SecretEncryptionKeyOutputFinder } from '@aws-accelerator/common-outputs/src/secrets'; +import { AcmOutputFinder } from '@aws-accelerator/common-outputs/src/certificates'; +import { SsmKmsOutputFinder } from '@aws-accelerator/common-outputs/src/ssm'; + +interface KmsOutput { + id: string; + name: string; + arn: string; +} + +interface AcmOutput { + name: string; + arn: string; +} + +export async function saveKmsKeys( + config: AcceleratorConfig, + outputs: StackOutput[], + ssm: SSM, + account: Account, + region: string, + acceleratorPrefix: string, + kms: OutputUtilGenericType[], +): Promise { + const kmsIndices = kms.flatMap(r => r.index) || []; + console.log('kmsIndices', kmsIndices); + let kmsMaxIndex = kmsIndices.length === 0 ? 0 : Math.max(...kmsIndices); + const updatedKeys: OutputUtilGenericType[] = []; + const removalObjects: OutputUtilGenericType[] = [...(kms || [])]; + + const masterAccount = config['global-options']['aws-org-master'].account; + const smRegion = config['global-options']['aws-org-master'].region; + const logAccount = config['global-options']['central-log-services'].account; + + const kmsOutputs: KmsOutput[] = []; + + if (region === smRegion) { + if (account.key !== logAccount) { + const accountBuckets = AccountBucketOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + kmsOutputs.push( + ...accountBuckets.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + } else { + const logBuckets = LogBucketOutputTypeOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + kmsOutputs.push( + ...logBuckets.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + } + + if (account.key === masterAccount) { + const centralBuckets = CentralBucketOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + kmsOutputs.push( + ...centralBuckets.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + + const secretKeys = SecretEncryptionKeyOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + kmsOutputs.push( + ...secretKeys.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + } + } + + const ebsKeys = EbsKmsOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + kmsOutputs.push( + ...ebsKeys.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + + const ssmKeys = SsmKmsOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + kmsOutputs.push( + ...ssmKeys.map(a => ({ + id: a.encryptionKeyId, + name: a.encryptionKeyName, + arn: a.encryptionKeyArn, + })), + ); + + for (const kmsOutput of kmsOutputs) { + console.log('kmsOutput', kmsOutput); + let currentIndex: number; + const previousGroupIndexDetails = kms.findIndex(p => p.name === kmsOutput.name); + if (previousGroupIndexDetails >= 0) { + currentIndex = kms[previousGroupIndexDetails].index; + console.log(`skipping creation of kms ${kmsOutput.name} in SSM`); + } else { + currentIndex = ++kmsMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/encrypt/kms/${currentIndex}/alias`, `${kmsOutput.name}`); + await ssm.putParameter(`/${acceleratorPrefix}/encrypt/kms/${currentIndex}/id`, kmsOutput.id); + await ssm.putParameter(`/${acceleratorPrefix}/encrypt/kms/${currentIndex}/arn`, `${kmsOutput.arn}`); + kms.push({ + name: kmsOutput.name, + index: currentIndex, + }); + } + updatedKeys.push({ + index: currentIndex, + name: kmsOutput.name, + }); + + const removalIndex = removalObjects.findIndex(p => p.name === kmsOutput.name); + if (removalIndex !== -1) { + removalObjects.splice(removalIndex, 1); + } + } + + for (const removeObject of removalObjects || []) { + const removalKms = [ + `/${acceleratorPrefix}/encrypt/kms/${removeObject.index}/alias`, + `/${acceleratorPrefix}/encrypt/kms/${removeObject.index}/id`, + `/${acceleratorPrefix}/encrypt/kms/${removeObject.index}/arn`, + ].flatMap(s => s); + + while (removalKms.length > 0) { + await ssm.deleteParameters(removalKms.splice(0, 10)); + } + } + return updatedKeys; +} + +export async function saveAcm( + config: AcceleratorConfig, + outputs: StackOutput[], + ssm: SSM, + account: Account, + region: string, + acceleratorPrefix: string, + acm: OutputUtilGenericType[], +): Promise { + const acmIndices = acm.flatMap(r => r.index) || []; + console.log('acmIndices', acmIndices); + let acmMaxIndex = acmIndices.length === 0 ? 0 : Math.max(...acmIndices); + const updatedAcm: OutputUtilGenericType[] = []; + const removalObjects: OutputUtilGenericType[] = [...(acm || [])]; + + const acmOutputs: AcmOutput[] = []; + + const acmCerts = AcmOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + acmOutputs.push( + ...acmCerts.map(a => ({ + name: a.certificateName, + arn: a.certificateArn, + })), + ); + + for (const acmOutput of acmOutputs) { + console.log('acmOutput', acmOutput); + let currentIndex: number; + const previousGroupIndexDetails = acm.findIndex(p => p.name === acmOutput.name); + if (previousGroupIndexDetails >= 0) { + currentIndex = acm[previousGroupIndexDetails].index; + console.log(`skipping creation of acm ${acmOutput.name} in SSM`); + } else { + currentIndex = ++acmMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/encrypt/acm/${currentIndex}/name`, `${acmOutput.name}`); + await ssm.putParameter(`/${acceleratorPrefix}/encrypt/acm/${currentIndex}/arn`, `${acmOutput.arn}`); + acm.push({ + name: acmOutput.name, + index: currentIndex, + }); + } + updatedAcm.push({ + index: currentIndex, + name: acmOutput.name, + }); + + const removalIndex = removalObjects.findIndex(p => p.name === acmOutput.name); + if (removalIndex !== -1) { + removalObjects.splice(removalIndex, 1); + } + } + + for (const removeObject of removalObjects || []) { + const removalAcm = [ + `/${acceleratorPrefix}/encrypt/acm/${removeObject.index}/name`, + `/${acceleratorPrefix}/encrypt/acm/${removeObject.index}/arn`, + ].flatMap(s => s); + + while (removalAcm.length > 0) { + await ssm.deleteParameters(removalAcm.splice(0, 10)); + } + } + return updatedAcm; +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 4354bb770..f022ac6b0 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -6,6 +6,7 @@ import { saveNetworkOutputs } from './network-outputs'; import { saveIamOutputs } from './iam-outputs'; import { saveElbOutputs } from './elb-outputs'; import { saveEventOutputs } from './event-outputs'; +import { saveEncryptsOutputs } from './encrypt-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -100,6 +101,17 @@ export const handler = async (input: SaveOutputsToSsmInput) => { region, }); + await saveEncryptsOutputs({ + acceleratorPrefix, + account, + assumeRoleName, + config, + dynamodb, + outputUtilsTableName, + outputsTableName, + region, + }); + return { status: 'SUCCESS', }; diff --git a/src/deployments/cdk/src/deployments/certificates/outputs.ts b/src/deployments/cdk/src/deployments/certificates/outputs.ts index 0734272a7..5f4bc4fb4 100644 --- a/src/deployments/cdk/src/deployments/certificates/outputs.ts +++ b/src/deployments/cdk/src/deployments/certificates/outputs.ts @@ -1,29 +1,8 @@ -import * as t from 'io-ts'; -import { createStructuredOutputFinder } from '@aws-accelerator/common-outputs/src/structured-output'; -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { createCfnStructuredOutput } from '../../common/structured-output'; +import { AcmOutput } from '@aws-accelerator/common-outputs/src/certificates'; export function createCertificateSecretName(certificateName: string): string { return `accelerator/certificates/${certificateName}`; } -export const AcmOutput = t.interface( - { - certificateName: t.string, - certificateArn: t.string, - }, - 'Acm', -); - -export type AcmOutput = t.TypeOf; - export const CfnAcmOutput = createCfnStructuredOutput(AcmOutput); - -export const AcmOutputFinder = createStructuredOutputFinder(AcmOutput, finder => ({ - findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => - finder.tryFindOne({ - outputs: props.outputs, - accountKey: props.accountKey, - region: props.region, - }), -})); diff --git a/src/deployments/cdk/src/deployments/defaults/outputs.ts b/src/deployments/cdk/src/deployments/defaults/outputs.ts index 4444545be..b4c154fa2 100644 --- a/src/deployments/cdk/src/deployments/defaults/outputs.ts +++ b/src/deployments/cdk/src/deployments/defaults/outputs.ts @@ -9,7 +9,6 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { StructuredOutput, createCfnStructuredOutput } from '../../common/structured-output'; import { EbsKmsOutput } from '@aws-accelerator/common-outputs/src/ebs'; import { SsmKmsOutput } from '@aws-accelerator/common-outputs/src/ssm'; -import { createStructuredOutputFinder } from '@aws-accelerator/common-outputs/src/structured-output'; export const CfnEbsKmsOutput = createCfnStructuredOutput(EbsKmsOutput); @@ -96,33 +95,6 @@ export const CfnLogBucketOutput = createCfnStructuredOutput(LogBucketOutputType) export const CfnCentralBucketOutput = createCfnStructuredOutput(CentralBucketOutputType); export const CfnAesBucketOutput = createCfnStructuredOutput(AesBucketOutputType); -export const AccountBucketOutputFinder = createStructuredOutputFinder(AccountBucketOutputType, finder => ({ - findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => - finder.tryFindOne({ - outputs: props.outputs, - accountKey: props.accountKey, - region: props.region, - }), -})); - -export const LogBucketOutputTypeOutputFinder = createStructuredOutputFinder(LogBucketOutputType, finder => ({ - findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => - finder.tryFindOne({ - outputs: props.outputs, - accountKey: props.accountKey, - region: props.region, - }), -})); - -export const CentralBucketOutputFinder = createStructuredOutputFinder(CentralBucketOutputType, finder => ({ - findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => - finder.tryFindOne({ - outputs: props.outputs, - accountKey: props.accountKey, - region: props.region, - }), -})); - export namespace AccountBucketOutput { /** * Helper method to import the account buckets from different phases. It includes the log bucket. diff --git a/src/lib/common-outputs/src/buckets.ts b/src/lib/common-outputs/src/buckets.ts new file mode 100644 index 000000000..a44a0bce7 --- /dev/null +++ b/src/lib/common-outputs/src/buckets.ts @@ -0,0 +1,72 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; +import { StackOutput } from './stack-output'; + +const AccountBucketOutput = t.interface( + { + bucketName: t.string, + bucketArn: t.string, + encryptionKeyArn: t.string, + region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, + }, + 'AccountBucket', +); + +type AccountBucketOutput = t.TypeOf; + +const LogBucketOutput = t.interface( + { + bucketName: t.string, + bucketArn: t.string, + encryptionKeyArn: t.string, + region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, + }, + 'LogBucket', +); + +type LogBucketOutput = t.TypeOf; + +const CentralBucketOutput = t.interface( + { + bucketName: t.string, + bucketArn: t.string, + encryptionKeyArn: t.string, + region: t.string, + encryptionKeyName: t.string, + encryptionKeyId: t.string, + }, + 'CentralBucket', +); + +type CentralBucketOutput = t.TypeOf; + +export const AccountBucketOutputFinder = createStructuredOutputFinder(AccountBucketOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); + +export const LogBucketOutputTypeOutputFinder = createStructuredOutputFinder(LogBucketOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); + +export const CentralBucketOutputFinder = createStructuredOutputFinder(CentralBucketOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); diff --git a/src/lib/common-outputs/src/certificates.ts b/src/lib/common-outputs/src/certificates.ts new file mode 100644 index 000000000..98b037daa --- /dev/null +++ b/src/lib/common-outputs/src/certificates.ts @@ -0,0 +1,22 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; +import { StackOutput } from './stack-output'; + +export const AcmOutput = t.interface( + { + certificateName: t.string, + certificateArn: t.string, + }, + 'Acm', +); + +export type AcmOutput = t.TypeOf; + +export const AcmOutputFinder = createStructuredOutputFinder(AcmOutput, finder => ({ + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + }), +})); diff --git a/src/lib/common-outputs/src/secrets.ts b/src/lib/common-outputs/src/secrets.ts index 033417bb5..6f3bc2f90 100644 --- a/src/lib/common-outputs/src/secrets.ts +++ b/src/lib/common-outputs/src/secrets.ts @@ -1,3 +1,6 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; + /** * Remove special characters from the start and end of a string. */ @@ -12,3 +15,16 @@ export function createFixedSecretName(props: { acceleratorPrefix: string; parts: const { acceleratorPrefix, parts } = props; return [trimSpecialCharacters(acceleratorPrefix), ...parts].join('/'); } + +export const SecretEncryptionKeyOutput = t.interface( + { + encryptionKeyName: t.string, + encryptionKeyId: t.string, + encryptionKeyArn: t.string, + }, + 'SecretEncryptionKeyOutput', +); + +export type SecretEncryptionKeyOutput = t.TypeOf; + +export const SecretEncryptionKeyOutputFinder = createStructuredOutputFinder(SecretEncryptionKeyOutput, () => ({})); From 820cddb9c8abb94902a9da311d149c5768696de0 Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 23 Sep 2020 20:46:47 +0530 Subject: [PATCH 18/25] added comments --- src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts | 7 +++++++ src/core/runtime/src/save-outputs-to-ssm/index.ts | 1 + 2 files changed, 8 insertions(+) diff --git a/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts index eb4180c8d..b6d59f709 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/encrypt-utils.ts @@ -45,7 +45,9 @@ export async function saveKmsKeys( const kmsOutputs: KmsOutput[] = []; + // Below outputs will be created only in SM region if (region === smRegion) { + // Finding account default bucket KMS keys in other accounts (excluding Log Archive account) if (account.key !== logAccount) { const accountBuckets = AccountBucketOutputFinder.findAll({ outputs, @@ -60,6 +62,7 @@ export async function saveKmsKeys( })), ); } else { + // Finding account bucket KMS key if it is Log Archive account const logBuckets = LogBucketOutputTypeOutputFinder.findAll({ outputs, accountKey: account.key, @@ -74,6 +77,7 @@ export async function saveKmsKeys( ); } + // If it is master account, checking Central Bucket and Secrets Kms Keys if (account.key === masterAccount) { const centralBuckets = CentralBucketOutputFinder.findAll({ outputs, @@ -105,6 +109,7 @@ export async function saveKmsKeys( } } + // Finding EBS KMS keys for the account const ebsKeys = EbsKmsOutputFinder.findAll({ outputs, accountKey: account.key, @@ -119,6 +124,7 @@ export async function saveKmsKeys( })), ); + // Finding SSM KMS keys for the account const ssmKeys = SsmKmsOutputFinder.findAll({ outputs, accountKey: account.key, @@ -192,6 +198,7 @@ export async function saveAcm( const acmOutputs: AcmOutput[] = []; + // Finding ACM certificates for the account const acmCerts = AcmOutputFinder.findAll({ outputs, accountKey: account.key, diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index f022ac6b0..48ea10d5c 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -101,6 +101,7 @@ export const handler = async (input: SaveOutputsToSsmInput) => { region, }); + // Store Encrypt outputs to SSM Parameter Store await saveEncryptsOutputs({ acceleratorPrefix, account, From 2eb72072a5b802bcfcac81621d9b34226be98ccc Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 24 Sep 2020 11:18:01 +0530 Subject: [PATCH 19/25] Updating SM for store outputs to ssm --- src/core/cdk/src/initial-setup.ts | 36 ++++++- .../src/tasks/store-outputs-to-ssm-task.ts | 97 +++++++++++++++++++ 2 files changed, 130 insertions(+), 3 deletions(-) create mode 100644 src/core/cdk/src/tasks/store-outputs-to-ssm-task.ts diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 9998fa79b..a8eab4966 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -21,6 +21,7 @@ import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task'; import * as fs from 'fs'; import * as sns from '@aws-cdk/aws-sns'; import { StoreOutputsTask } from './tasks/store-outputs-task'; +import { StoreOutputsToSSMTask } from './tasks/store-outputs-to-ssm-task'; export namespace InitialSetup { export interface CommonProps { @@ -511,7 +512,36 @@ export namespace InitialSetup { resultPath: 'DISCARD', }); - const pass = new sfn.Pass(this, 'Success'); + const storeOutputsToSsmStateMachine = new sfn.StateMachine( + this, + `${props.acceleratorPrefix}StoreOutputsToSsm_sm`, + { + stateMachineName: `${props.acceleratorPrefix}StoreOutputsToSsm_sm`, + definition: new StoreOutputsToSSMTask(this, 'StoreOutputsToSSM', { + lambdaCode, + role: pipelineRole, + }), + }, + ); + + const storeAllOutputsToSsmTask = new sfn.Task(this, 'Store Outputs to SSM', { + // tslint:disable-next-line: deprecation + task: new tasks.StartExecution(storeOutputsToSsmStateMachine, { + integrationPattern: sfn.ServiceIntegrationPattern.SYNC, + input: { + 'accounts.$': '$.accounts', + 'regions.$': '$.regions', + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTableName: outputsTable.tableName, + configRepositoryName: props.configRepositoryName, + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', + outputUtilsTableName: outputUtilsTable.tableName, + }, + }), + resultPath: 'DISCARD', + }); const detachQuarantineScpTask = new CodeTask(this, 'Detach Quarantine SCP', { functionProps: { @@ -525,7 +555,7 @@ export namespace InitialSetup { }, resultPath: 'DISCARD', }); - detachQuarantineScpTask.next(pass); + detachQuarantineScpTask.next(storeAllOutputsToSsmTask); const enableTrustedAccessForServicesTask = new CodeTask(this, 'Enable Trusted Access For Services', { functionProps: { @@ -802,7 +832,7 @@ export namespace InitialSetup { const baseLineCleanupChoice = new sfn.Choice(this, 'Baseline Clean Up?') .when(sfn.Condition.stringEquals('$.baseline', 'ORGANIZATIONS'), detachQuarantineScpTask) - .otherwise(pass); + .otherwise(storeAllOutputsToSsmTask); const commonStep1 = addScpTask.startState .next(deployPhase1Task) diff --git a/src/core/cdk/src/tasks/store-outputs-to-ssm-task.ts b/src/core/cdk/src/tasks/store-outputs-to-ssm-task.ts new file mode 100644 index 000000000..a7b5711a0 --- /dev/null +++ b/src/core/cdk/src/tasks/store-outputs-to-ssm-task.ts @@ -0,0 +1,97 @@ +import * as cdk from '@aws-cdk/core'; +import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { CodeTask } from '@aws-accelerator/cdk-accelerator/src/stepfunction-tasks'; + +export namespace StoreOutputsToSSMTask { + export interface Props { + role: iam.IRole; + lambdaCode: lambda.Code; + functionPayload?: { [key: string]: unknown }; + waitSeconds?: number; + } +} + +export class StoreOutputsToSSMTask extends sfn.StateMachineFragment { + readonly startState: sfn.State; + readonly endStates: sfn.INextable[]; + + constructor(scope: cdk.Construct, id: string, props: StoreOutputsToSSMTask.Props) { + super(scope, id); + + const { role, lambdaCode, functionPayload, waitSeconds = 10 } = props; + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + resources: ['*'], + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + }), + ); + + const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs To SSM`, { + itemsPath: `$.accounts`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'accountId.$': '$$.Map.Item.Value', + 'regions.$': '$.regions', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTableName.$': '$.outputsTableName', + 'configRepositoryName.$': '$.configRepositoryName', + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', + 'outputUtilsTableName.$': '$.outputUtilsTableName', + }, + }); + + const getAccountInfoTask = new CodeTask(scope, `Get Account Details`, { + comment: 'Get Account Info', + resultPath: '$.account', + functionPayload, + functionProps: { + role, + code: lambdaCode, + handler: 'index.getAccountInfo', + }, + }); + + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs To SSM`, { + itemsPath: `$.regions`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$.account', + 'region.$': '$$.Map.Item.Value', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTableName.$': '$.outputsTableName', + 'configRepositoryName.$': '$.configRepositoryName', + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', + 'outputUtilsTableName.$': '$.outputUtilsTableName', + }, + }); + + getAccountInfoTask.next(storeAccountRegionOutputs); + const storeOutputsTask = new CodeTask(scope, `Store Outputs To SSM`, { + resultPath: '$.storeOutputsOutput', + functionPayload, + functionProps: { + role, + code: lambdaCode, + handler: 'index.saveOutputsToSSM', + }, + }); + + const pass = new sfn.Pass(this, 'Store Outputs To SSM Success'); + storeAccountOutputs.iterator(getAccountInfoTask); + storeAccountRegionOutputs.iterator(storeOutputsTask); + const chain = sfn.Chain.start(storeAccountOutputs).next(pass); + + this.startState = chain.startState; + this.endStates = chain.endStates; + } +} From cea6afbf4bb219eb0514a206a9533deb2e9890e7 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 24 Sep 2020 13:52:11 +0530 Subject: [PATCH 20/25] Increasing SSM Parameter Store Throughput in all supported regions --- src/deployments/cdk/package.json | 1 + src/deployments/cdk/src/apps/phase--1.ts | 7 +++ src/deployments/cdk/src/apps/phase-5.ts | 12 ++++ .../cdk/src/deployments/iam/index.ts | 1 + .../deployments/iam/ssm-throughput-roles.ts | 45 ++++++++++++++ .../increase-parameter-store-throughput.ts | 43 +++++++++++++ .../cdk/src/deployments/ssm/index.ts | 1 + .../cdk-ssm-increase-throughput/README.md | 11 ++++ .../cdk-ssm-increase-throughput/cdk/index.ts | 60 +++++++++++++++++++ .../cdk-ssm-increase-throughput/package.json | 25 ++++++++ .../cdk-ssm-increase-throughput/tsconfig.json | 21 +++++++ 11 files changed, 227 insertions(+) create mode 100644 src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts create mode 100644 src/deployments/cdk/src/deployments/ssm/increase-parameter-store-throughput.ts create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/README.md create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/package.json create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/tsconfig.json diff --git a/src/deployments/cdk/package.json b/src/deployments/cdk/package.json index 6fb9e80da..28ddb38a7 100644 --- a/src/deployments/cdk/package.json +++ b/src/deployments/cdk/package.json @@ -83,6 +83,7 @@ "@aws-accelerator/custom-resource-associate-hosted-zones": "workspace:^0.0.1", "@aws-accelerator/custom-resource-associate-resolver-rules": "workspace:^0.0.1", "@aws-accelerator/custom-resource-create-resolver-rule": "workspace:^0.0.1", + "@aws-accelerator/custom-resource-ssm-increase-throughput": "workspace:^0.0.1", "@aws-cdk/aws-accessanalyzer": "1.46.0", "@aws-cdk/aws-autoscaling": "1.46.0", "@aws-cdk/aws-budgets": "1.46.0", diff --git a/src/deployments/cdk/src/apps/phase--1.ts b/src/deployments/cdk/src/apps/phase--1.ts index 9c7fdaa73..9395a05cf 100644 --- a/src/deployments/cdk/src/apps/phase--1.ts +++ b/src/deployments/cdk/src/apps/phase--1.ts @@ -16,6 +16,7 @@ import * as globalRoles from '../deployments/iam'; * - Creating required roles for TransitGatewayAcceptPeeringAttachment custom resource * - Creating required roles for createLogsMetricFilter custom resource * - Creating required roles for SnsSubscriberLambda custom resource + * - Creating required role for SsmIncreaseThroughput custom resource */ export async function deploy({ acceleratorConfig, accountStacks, accounts }: PhaseInput) { // creates roles for macie custom resources @@ -98,4 +99,10 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts }: Pha accountStacks, config: acceleratorConfig, }); + + // Creates required role for SsmIncreaseThroughput custom resource + await globalRoles.createSsmThroughputRole({ + accountStacks, + accounts, + }); } diff --git a/src/deployments/cdk/src/apps/phase-5.ts b/src/deployments/cdk/src/apps/phase-5.ts index f61b14118..3be41e5b3 100644 --- a/src/deployments/cdk/src/apps/phase-5.ts +++ b/src/deployments/cdk/src/apps/phase-5.ts @@ -14,6 +14,7 @@ import * as cwlCentralLoggingToS3 from '../deployments/central-services/central- import { ArtifactOutputFinder } from '../deployments/artifacts/outputs'; import { ImageIdOutputFinder } from '@aws-accelerator/common-outputs/src/ami-output'; import * as cloudWatchDeployment from '../deployments/cloud-watch'; +import * as ssmDeployment from '../deployments/ssm'; /** * This is the main entry point to deploy phase 5 @@ -24,6 +25,7 @@ import * as cloudWatchDeployment from '../deployments/cloud-watch'; * - enable central logging to S3 (step 2) * - Create CloudWatch Events for moveAccount, policyChanges and createAccount. * - Creates CloudWatch Alarms + * - Increase SSM Parameter Store throughput in all accounts and supported regions */ export async function deploy({ acceleratorConfig, accountStacks, accounts, context, outputs }: PhaseInput) { @@ -193,4 +195,14 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte config: acceleratorConfig, accounts, }); + + /** + * Increasing SSM Parameter Store throughput in all accounts and supported regions + */ + await ssmDeployment.step2({ + accountStacks, + accounts, + config: acceleratorConfig, + outputs, + }); } diff --git a/src/deployments/cdk/src/deployments/iam/index.ts b/src/deployments/cdk/src/deployments/iam/index.ts index 252a4ea6a..0b41de511 100644 --- a/src/deployments/cdk/src/deployments/iam/index.ts +++ b/src/deployments/cdk/src/deployments/iam/index.ts @@ -15,3 +15,4 @@ export * from './logs-metric-filter-role'; export * from './sns-subscriber-lambda-role'; export * from './cleanup-role'; export * from './central-endpoints-deployment-roles'; +export * from './ssm-throughput-roles'; diff --git a/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts b/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts new file mode 100644 index 000000000..13c455e2a --- /dev/null +++ b/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts @@ -0,0 +1,45 @@ +import * as iam from '@aws-cdk/aws-iam'; +import * as cdk from '@aws-cdk/core'; +import { AccountStacks, AccountStack } from '../../common/account-stacks'; +import { createIamRoleOutput } from './outputs'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; + +export interface CreateSsmThroughputRoleProps { + accountStacks: AccountStacks; + accounts: Account[]; +} + +export async function createSsmThroughputRole(props: CreateSsmThroughputRoleProps): Promise { + const { accountStacks, accounts } = props; + for (const account of accounts) { + const accountStack = accountStacks.tryGetOrCreateAccountStack(account.key); + if (!accountStack) { + console.warn(`Unable to create Account Stack for Account "${account.key}"`); + continue; + } + const iamRole = await createRole(accountStack); + createIamRoleOutput(accountStack, iamRole, 'SSMUpdateRole'); + } +} + +async function createRole(stack: AccountStack) { + const role = new iam.Role(stack, 'Custom::SSMUpdateRole', { + assumedBy: new iam.ServicePrincipal('lambda.amazonaws.com'), + }); + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + resources: ['*'], + }), + ); + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + actions: ['ssm:GetServiceSetting'], + resources: ['*'], + }), + ); + + return role; +} diff --git a/src/deployments/cdk/src/deployments/ssm/increase-parameter-store-throughput.ts b/src/deployments/cdk/src/deployments/ssm/increase-parameter-store-throughput.ts new file mode 100644 index 000000000..adea0d9ba --- /dev/null +++ b/src/deployments/cdk/src/deployments/ssm/increase-parameter-store-throughput.ts @@ -0,0 +1,43 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { AcceleratorConfig } from '@aws-accelerator/common-config/src'; +import { AccountStacks } from '../../common/account-stacks'; +import { Account } from '../../utils/accounts'; +import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; +import { SsmIncreaseThroughput } from '@aws-accelerator/custom-resource-ssm-increase-throughput'; + +export interface SSMStep2Props { + accountStacks: AccountStacks; + config: AcceleratorConfig; + accounts: Account[]; + outputs: StackOutput[]; +} + +/** + * Increasing SSM Parameter store throughput + * @param props + */ +export async function step2(props: SSMStep2Props) { + const { accountStacks, accounts, config, outputs } = props; + const regions = config['global-options']['supported-regions']; + for (const account of accounts) { + const ssmUpdateRole = IamRoleOutputFinder.tryFindOneByName({ + outputs, + accountKey: account.key, + roleKey: 'SSMUpdateRole', + }); + if (!ssmUpdateRole) { + console.warn(`No role created for "${account.key}"`); + continue; + } + for (const region of regions) { + const accountStack = accountStacks.tryGetOrCreateAccountStack(account.key, region); + if (!accountStack) { + console.warn(`Unable to create Account Stak for Account "${account.key}" and Region "${region}"`); + continue; + } + new SsmIncreaseThroughput(accountStack, 'UpdateSSMParameterStoreThroughput', { + roleArn: ssmUpdateRole.roleArn, + }); + } + } +} diff --git a/src/deployments/cdk/src/deployments/ssm/index.ts b/src/deployments/cdk/src/deployments/ssm/index.ts index 04100bb64..d6c349d95 100644 --- a/src/deployments/cdk/src/deployments/ssm/index.ts +++ b/src/deployments/cdk/src/deployments/ssm/index.ts @@ -1 +1,2 @@ export * from './session-manager'; +export * from './increase-parameter-store-throughput'; diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/README.md b/src/lib/custom-resources/cdk-ssm-increase-throughput/README.md new file mode 100644 index 000000000..51862c183 --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/README.md @@ -0,0 +1,11 @@ +# SSM Increase Throughput + +This is a custom resource to set limit for SSM Parameter Store Throughput `updateServiceSetting` API call. + +## Usage + + import { SsmIncreaseThroughput } from '@aws-accelerator/custom-resource-ssm-increase-throughput'; + + new SsmIncreaseThroughput(accountStack, 'UpdateSSMParameterStoreThroughput', { + roleArn: ssmUpdateRole.roleArn, + ); diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts b/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts new file mode 100644 index 000000000..65cd2a433 --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts @@ -0,0 +1,60 @@ +import * as cdk from '@aws-cdk/core'; +import * as custom from '@aws-cdk/custom-resources'; +import * as iam from '@aws-cdk/aws-iam'; + +const resourceType = 'Custom::SSMIncreaseThroughput'; + +export interface SsmIncreaseThroughputProps { + roleArn: string; +} + +/** + * Custom resource implementation that create logs resource policy. Awaiting + * https://github.com/aws-cloudformation/aws-cloudformation-coverage-roadmap/issues/249 + */ +export class SsmIncreaseThroughput extends cdk.Construct { + private role: iam.IRole; + constructor(scope: cdk.Construct, id: string, props: SsmIncreaseThroughputProps) { + super(scope, id); + + this.role = iam.Role.fromRoleArn(this, `${resourceType}Role`, props.roleArn); + + const physicalResourceId = custom.PhysicalResourceId.of('SSMParameterStoreIncreaseThroughput'); + + const onCreateOrUpdate: custom.AwsSdkCall = { + service: 'SSM', + action: 'updateServiceSetting', + physicalResourceId, + parameters: { + SettingId: '/ssm/parameter-store/high-throughput-enabled', + SettingValue: 'true', + }, + }; + + const onDelete: custom.AwsSdkCall = { + service: 'SSM', + action: 'updateServiceSetting', + physicalResourceId, + parameters: { + SettingId: '/ssm/parameter-store/high-throughput-enabled', + SettingValue: 'false', + }, + }; + + new custom.AwsCustomResource(this, 'Resource', { + resourceType: resourceType, + onCreate: onCreateOrUpdate, + onUpdate: onCreateOrUpdate, + onDelete: onDelete, + policy: custom.AwsCustomResourcePolicy.fromStatements([ + new iam.PolicyStatement({ + actions: ['ssm:UpdateServiceSetting', 'ssm:ResetServiceSetting'], + resources: [ + `arn:aws:ssm:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:servicesetting/ssm/parameter-store/high-throughput-enabled`, + ], + }), + ]), + role: this.role, + }); + } +} diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json b/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json new file mode 100644 index 000000000..a118ee85c --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json @@ -0,0 +1,25 @@ +{ + "name": "@aws-accelerator/custom-resource-ssm-increase-throughput", + "peerDependencies": { + "@aws-cdk/aws-iam": "^1.46.0", + "@aws-cdk/core": "^1.46.0", + "@aws-cdk/custom-resources": "^1.46.0" + }, + "main": "cdk/index.ts", + "private": true, + "version": "0.0.1", + "dependencies": { + "@aws-cdk/aws-iam": "1.46.0", + "@aws-cdk/core": "1.46.0", + "@aws-cdk/custom-resources": "1.46.0" + }, + "devDependencies": { + "tslint": "6.1.0", + "typescript": "3.8.3", + "tslint-config-standard": "9.0.0", + "tslint-config-prettier": "1.18.0", + "ts-node": "6.2.0", + "@types/node": "12.12.6" + } + +} \ No newline at end of file diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/tsconfig.json b/src/lib/custom-resources/cdk-ssm-increase-throughput/tsconfig.json new file mode 100644 index 000000000..4db940b9b --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "es2019", + "lib": [ + "es2019" + ], + "noImplicitAny": true, + "module": "commonjs", + "strict": true, + "esModuleInterop": true, + "moduleResolution": "node", + "outDir": "dist" + }, + "exclude": [ + "node_modules", + "**/*.spec.ts" + ], + "include": [ + "cdk/**/*" + ] +} \ No newline at end of file From a6d37a2da8f3f4fa6fe59e828972a23323cc55b3 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 24 Sep 2020 15:43:07 +0530 Subject: [PATCH 21/25] Fixing ssh throughput incresing --- .../deployments/iam/ssm-throughput-roles.ts | 6 ++ .../cdk-ssm-increase-throughput/cdk/index.ts | 61 ++++++++----------- .../cdk-ssm-increase-throughput/package.json | 9 +-- .../runtime/.gitignore | 1 + .../runtime/package.json | 31 ++++++++++ .../runtime/src/index.ts | 61 +++++++++++++++++++ .../runtime/tsconfig.json | 15 +++++ .../runtime/webpack.config.ts | 4 ++ 8 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/.gitignore create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/package.json create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/src/index.ts create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/tsconfig.json create mode 100644 src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/webpack.config.ts diff --git a/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts b/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts index 13c455e2a..a9e83ac2c 100644 --- a/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts +++ b/src/deployments/cdk/src/deployments/iam/ssm-throughput-roles.ts @@ -41,5 +41,11 @@ async function createRole(stack: AccountStack) { }), ); + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + actions: ['ssm:UpdateServiceSetting', 'ssm:ResetServiceSetting'], + resources: [`arn:aws:ssm:*:${cdk.Aws.ACCOUNT_ID}:servicesetting/ssm/parameter-store/high-throughput-enabled`], + }), + ); return role; } diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts b/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts index 65cd2a433..b11e03374 100644 --- a/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/cdk/index.ts @@ -1,6 +1,7 @@ +import * as path from 'path'; import * as cdk from '@aws-cdk/core'; -import * as custom from '@aws-cdk/custom-resources'; import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; const resourceType = 'Custom::SSMIncreaseThroughput'; @@ -14,47 +15,33 @@ export interface SsmIncreaseThroughputProps { */ export class SsmIncreaseThroughput extends cdk.Construct { private role: iam.IRole; + private readonly resource: cdk.CustomResource; constructor(scope: cdk.Construct, id: string, props: SsmIncreaseThroughputProps) { super(scope, id); - this.role = iam.Role.fromRoleArn(this, `${resourceType}Role`, props.roleArn); + this.resource = new cdk.CustomResource(this, 'Resource', { + resourceType, + serviceToken: this.lambdaFunction.functionArn, + }); + } - const physicalResourceId = custom.PhysicalResourceId.of('SSMParameterStoreIncreaseThroughput'); - - const onCreateOrUpdate: custom.AwsSdkCall = { - service: 'SSM', - action: 'updateServiceSetting', - physicalResourceId, - parameters: { - SettingId: '/ssm/parameter-store/high-throughput-enabled', - SettingValue: 'true', - }, - }; - - const onDelete: custom.AwsSdkCall = { - service: 'SSM', - action: 'updateServiceSetting', - physicalResourceId, - parameters: { - SettingId: '/ssm/parameter-store/high-throughput-enabled', - SettingValue: 'false', - }, - }; - - new custom.AwsCustomResource(this, 'Resource', { - resourceType: resourceType, - onCreate: onCreateOrUpdate, - onUpdate: onCreateOrUpdate, - onDelete: onDelete, - policy: custom.AwsCustomResourcePolicy.fromStatements([ - new iam.PolicyStatement({ - actions: ['ssm:UpdateServiceSetting', 'ssm:ResetServiceSetting'], - resources: [ - `arn:aws:ssm:${cdk.Aws.REGION}:${cdk.Aws.ACCOUNT_ID}:servicesetting/ssm/parameter-store/high-throughput-enabled`, - ], - }), - ]), + private get lambdaFunction(): lambda.Function { + const constructName = `${resourceType}Lambda`; + const stack = cdk.Stack.of(this); + const existing = stack.node.tryFindChild(constructName); + if (existing) { + return existing as lambda.Function; + } + + const lambdaPath = require.resolve('@aws-accelerator/custom-resource-ssm-increase-throughput-runtime'); + const lambdaDir = path.dirname(lambdaPath); + + return new lambda.Function(stack, constructName, { + runtime: lambda.Runtime.NODEJS_12_X, + code: lambda.Code.fromAsset(lambdaDir), + handler: 'index.handler', role: this.role, + timeout: cdk.Duration.minutes(15), }); } } diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json b/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json index a118ee85c..3864461b4 100644 --- a/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/package.json @@ -11,15 +11,16 @@ "dependencies": { "@aws-cdk/aws-iam": "1.46.0", "@aws-cdk/core": "1.46.0", - "@aws-cdk/custom-resources": "1.46.0" + "@aws-cdk/aws-lambda": "1.46.0" }, "devDependencies": { "tslint": "6.1.0", - "typescript": "3.8.3", "tslint-config-standard": "9.0.0", + "@types/aws-lambda": "8.10.46", "tslint-config-prettier": "1.18.0", - "ts-node": "6.2.0", - "@types/node": "12.12.6" + "@types/cfn-response": "1.0.3", + "@types/node": "12.12.6", + "@aws-accelerator/custom-resource-ssm-increase-throughput-runtime": "workspace:^0.0.1" } } \ No newline at end of file diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/.gitignore b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/.gitignore new file mode 100644 index 000000000..1521c8b76 --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/.gitignore @@ -0,0 +1 @@ +dist diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/package.json b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/package.json new file mode 100644 index 000000000..56fa1b65a --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/package.json @@ -0,0 +1,31 @@ +{ + "name": "@aws-accelerator/custom-resource-ssm-increase-throughput-runtime", + "externals": [ + "aws-lambda", + "aws-sdk" + ], + "private": true, + "source": "src/index.ts", + "version": "0.0.1", + "dependencies": { + "@aws-accelerator/custom-resource-runtime-cfn-response": "workspace:^0.0.1", + "@aws-accelerator/custom-resource-cfn-utils": "workspace:^0.0.1", + "exponential-backoff": "3.0.0", + "aws-sdk": "2.668.0", + "aws-lambda": "1.0.5" + }, + "scripts": { + "prepare": "webpack-cli --config webpack.config.ts" + }, + "devDependencies": { + "ts-loader": "7.0.5", + "typescript": "3.8.3", + "@types/aws-lambda": "8.10.46", + "webpack": "4.42.1", + "@aws-accelerator/custom-resource-runtime-webpack-base": "workspace:^0.0.1", + "@types/node": "12.12.6", + "webpack-cli": "3.3.11" + }, + "main": "dist/index.js", + "types": "dist/index.d.ts" +} \ No newline at end of file diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/src/index.ts b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/src/index.ts new file mode 100644 index 000000000..69db02f17 --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/src/index.ts @@ -0,0 +1,61 @@ +import * as AWS from 'aws-sdk'; +AWS.config.logger = console; +import { CloudFormationCustomResourceDeleteEvent, CloudFormationCustomResourceEvent } from 'aws-lambda'; +import { errorHandler } from '@aws-accelerator/custom-resource-runtime-cfn-response'; +import { throttlingBackOff } from '@aws-accelerator/custom-resource-cfn-utils'; + +const ssm = new AWS.SSM(); + +export const handler = errorHandler(onEvent); + +async function onEvent(event: CloudFormationCustomResourceEvent) { + console.log(`Updating SSM Parameter Store throughput...`); + console.log(JSON.stringify(event, null, 2)); + + // tslint:disable-next-line: switch-default + switch (event.RequestType) { + case 'Create': + return onCreateOrUpdate(event); + case 'Update': + return onCreateOrUpdate(event); + case 'Delete': + return onDelete(event); + } +} + +async function onCreateOrUpdate(_: CloudFormationCustomResourceEvent) { + try { + await throttlingBackOff(() => + ssm + .updateServiceSetting({ + SettingId: '/ssm/parameter-store/high-throughput-enabled', + SettingValue: 'true', + }) + .promise(), + ); + } catch (error) { + console.warn('Error while setting limit to ssm parameter store'); + console.warn(error); + } + return { + physicalResourceId: `/ssm/parameter-store/high-throughput-enabled`, + }; +} + +async function onDelete(event: CloudFormationCustomResourceDeleteEvent) { + if (event.PhysicalResourceId === '/ssm/parameter-store/high-throughput-enabled') { + try { + await throttlingBackOff(() => + ssm + .updateServiceSetting({ + SettingId: '/ssm/parameter-store/high-throughput-enabled', + SettingValue: 'false', + }) + .promise(), + ); + } catch (error) { + console.warn('Error while setting limit to ssm parameter store'); + console.warn(error); + } + } +} diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/tsconfig.json b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/tsconfig.json new file mode 100644 index 000000000..118a8376a --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "es2019", + "lib": ["es2019"], + "module": "commonjs", + "moduleResolution": "node", + "strict": true, + "declaration": true, + "esModuleInterop": true, + "noImplicitAny": true, + "resolveJsonModule": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "**/*.spec.ts"] +} diff --git a/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/webpack.config.ts b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/webpack.config.ts new file mode 100644 index 000000000..425acd8ba --- /dev/null +++ b/src/lib/custom-resources/cdk-ssm-increase-throughput/runtime/webpack.config.ts @@ -0,0 +1,4 @@ +import { webpackConfigurationForPackage } from '@aws-accelerator/custom-resource-runtime-webpack-base'; +import pkg from './package.json'; + +export default webpackConfigurationForPackage(pkg); From 410aa02d96485e005f76a8c729ce612b0791f67f Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Thu, 24 Sep 2020 19:57:22 +0530 Subject: [PATCH 22/25] Saving Firewall outputs to SSM --- .../save-outputs-to-ssm/firewall-outputs.ts | 140 ++++++++++++++++++ .../runtime/src/save-outputs-to-ssm/index.ts | 13 ++ .../deployments/firewall/cluster/outputs.ts | 3 + .../deployments/firewall/cluster/step-3.ts | 20 ++- .../cdk-constructs/src/firewall/instance.ts | 7 +- src/lib/common-outputs/src/firewall.ts | 17 +++ .../cdk-s3-template/cdk/index.ts | 4 + 7 files changed, 202 insertions(+), 2 deletions(-) create mode 100644 src/core/runtime/src/save-outputs-to-ssm/firewall-outputs.ts create mode 100644 src/lib/common-outputs/src/firewall.ts diff --git a/src/core/runtime/src/save-outputs-to-ssm/firewall-outputs.ts b/src/core/runtime/src/save-outputs-to-ssm/firewall-outputs.ts new file mode 100644 index 000000000..6e01af3e0 --- /dev/null +++ b/src/core/runtime/src/save-outputs-to-ssm/firewall-outputs.ts @@ -0,0 +1,140 @@ +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getOutput, OutputUtilGenericType, SaveOutputsInput, getIndexOutput, saveIndexOutput } from './utils'; +import { SSM } from '@aws-accelerator/common/src/aws/ssm'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { FirewallConfigReplacementsOutputFinder } from '@aws-accelerator/common-outputs/src/firewall'; + +const OUTPUT_TYPE = 'firewall'; +interface OutputUtilFireallReplacements extends OutputUtilGenericType { + replacements: string[]; +} +interface OutputUtilFirewall { + firewalls?: OutputUtilFireallReplacements[]; +} + +/** + * Outputs for Firewall Replacement related deployments will be found in following phases + * - Phase-2 + */ + +/** + * + * @param outputsTableName + * @param client + * @param config + * @param account + * + * @returns void + */ +export async function saveFirewallReplacementOutputs(props: SaveOutputsInput) { + const { + acceleratorPrefix, + account, + config, + dynamodb, + outputsTableName, + assumeRoleName, + region, + outputUtilsTableName, + } = props; + + const dataKey = `${account.key}-${region}-firewall`; + const removeNames: string[] = []; + + const accountConfig = config.getAccountByKey(account.key); + if (!accountConfig.deployments || !accountConfig.deployments.firewalls) { + return; + } + + if (!accountConfig.deployments.firewalls.find(f => f.region === region)) { + return; + } + const oldFirewallOutputUtils = await getIndexOutput(outputUtilsTableName, dataKey, dynamodb); + // Existing index check happens on this variable + let outputUtils: OutputUtilFirewall; + if (oldFirewallOutputUtils) { + outputUtils = oldFirewallOutputUtils; + } else { + outputUtils = {}; + } + if (!outputUtils.firewalls) { + outputUtils.firewalls = []; + } + + // Storing new resource index and updating DDB in this variable + const newOutputUtils: OutputUtilFirewall = {}; + newOutputUtils.firewalls = []; + + const sts = new STS(); + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const ssm = new SSM(credentials, region); + + const outputs: StackOutput[] = await getOutput(outputsTableName, `${account.key}-${region}-2`, dynamodb); + const firewallOutputs = FirewallConfigReplacementsOutputFinder.findAll({ + outputs, + accountKey: account.key, + region, + }); + + const indices = outputUtils.firewalls.flatMap(e => e.index) || []; + let maxIndex = indices.length === 0 ? 0 : Math.max(...indices); + + for (const output of firewallOutputs) { + let currentIndex: number; + const previousIndex = outputUtils.firewalls.findIndex(f => f.name === output.instanceId); + if (previousIndex >= 0) { + currentIndex = outputUtils.firewalls[previousIndex].index; + } else { + currentIndex = ++maxIndex; + } + newOutputUtils.firewalls.push({ + index: currentIndex, + name: output.instanceId, + replacements: Object.keys(output.replacements), + }); + + if (previousIndex < 0) { + await ssm.putParameter( + `/${acceleratorPrefix}/${OUTPUT_TYPE}/${output.name}/${currentIndex}/name`, + `${output.name}`, + ); + for (const [key, value] of Object.entries(output.replacements)) { + await ssm.putParameter( + `/${acceleratorPrefix}/${OUTPUT_TYPE}/${output.name}/${currentIndex}/${key}`, + `${value}`, + ); + } + } else { + const previousReplacements = Object.keys(outputUtils.firewalls[previousIndex].replacements); + const currentReplacements = Object.keys(output.replacements); + for (const replacement of currentReplacements.filter(cr => !previousReplacements.includes(cr))) { + await ssm.putParameter( + `/${acceleratorPrefix}/${OUTPUT_TYPE}/${output.name}/${currentIndex}/${replacement}`, + `${output.replacements[replacement]}`, + ); + } + + removeNames.push( + ...previousReplacements + .filter(pr => !currentReplacements.includes(pr)) + .map(r => `/${acceleratorPrefix}/${OUTPUT_TYPE}/${output.name}/${currentIndex}/${r}`), + ); + } + if (previousIndex >= 0) { + outputUtils.firewalls.splice(previousIndex, 1); + } + } + + await saveIndexOutput(outputUtilsTableName, dataKey, JSON.stringify(newOutputUtils), dynamodb); + removeNames.push( + ...outputUtils.firewalls + .map(e => [ + `/${acceleratorPrefix}/${OUTPUT_TYPE}/${e.name}/${e.index}/name`, + ...e.replacements.map(r => `/${acceleratorPrefix}/${OUTPUT_TYPE}/${e.name}/${e.index}/${r}`), + ]) + .flatMap(es => es), + ); + while (removeNames.length > 0) { + await ssm.deleteParameters(removeNames.splice(0, 10)); + } +} diff --git a/src/core/runtime/src/save-outputs-to-ssm/index.ts b/src/core/runtime/src/save-outputs-to-ssm/index.ts index 48ea10d5c..337c48970 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/index.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/index.ts @@ -7,6 +7,7 @@ import { saveIamOutputs } from './iam-outputs'; import { saveElbOutputs } from './elb-outputs'; import { saveEventOutputs } from './event-outputs'; import { saveEncryptsOutputs } from './encrypt-outputs'; +import { saveFirewallReplacementOutputs } from './firewall-outputs'; export interface SaveOutputsToSsmInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -113,6 +114,18 @@ export const handler = async (input: SaveOutputsToSsmInput) => { region, }); + // Store Firewall Outputs to SSM Parameter Store + await saveFirewallReplacementOutputs({ + acceleratorPrefix, + account, + assumeRoleName, + config, + dynamodb, + outputUtilsTableName, + outputsTableName, + region, + }); + return { status: 'SUCCESS', }; diff --git a/src/deployments/cdk/src/deployments/firewall/cluster/outputs.ts b/src/deployments/cdk/src/deployments/firewall/cluster/outputs.ts index 38a282408..fbfc180b1 100644 --- a/src/deployments/cdk/src/deployments/firewall/cluster/outputs.ts +++ b/src/deployments/cdk/src/deployments/firewall/cluster/outputs.ts @@ -3,6 +3,7 @@ import { optional } from '@aws-accelerator/common-types'; import { createCfnStructuredOutput } from '../../../common/structured-output'; import { createStructuredOutputFinder } from '@aws-accelerator/common-outputs/src/structured-output'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { FirewallConfigReplacementsOutput } from '@aws-accelerator/common-outputs/src/firewall'; export const FirewallInstanceOutput = t.interface( { @@ -13,6 +14,8 @@ export const FirewallInstanceOutput = t.interface( 'FirewallInstanceOutput', ); +export const CfnFirewallConfigReplacementsOutput = createCfnStructuredOutput(FirewallConfigReplacementsOutput); + export type FirewallInstanceOutput = t.TypeOf; export const FirewallInstanceOutputFinder = createStructuredOutputFinder(FirewallInstanceOutput, () => ({})); diff --git a/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts b/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts index e59306a05..b62f2c233 100644 --- a/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts +++ b/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts @@ -12,7 +12,12 @@ import { } from '@aws-accelerator/common-outputs/src/stack-output'; import { FirewallCluster, FirewallInstance } from '@aws-accelerator/cdk-constructs/src/firewall'; import { AccountStacks, AccountStack } from '../../../common/account-stacks'; -import { FirewallVpnConnection, CfnFirewallInstanceOutput, FirewallVpnConnectionOutputFinder } from './outputs'; +import { + FirewallVpnConnection, + CfnFirewallInstanceOutput, + FirewallVpnConnectionOutputFinder, + CfnFirewallConfigReplacementsOutput, +} from './outputs'; import { checkAccountWarming } from '../../account-warming/outputs'; import { createIamInstanceProfileName } from '../../../common/iam-assets'; import { RegionalBucket } from '../../defaults'; @@ -254,5 +259,18 @@ async function createFirewallCluster(props: { } } } + + for (const instance of Object.values(instancePerAz)) { + const replacements: { [key: string]: string} = {}; + Object.entries(instance.replacements).forEach( ([key, value]) => { + replacements[key.replace(/[^-a-zA-Z0-9_.]+/gi, '')] = value; + }); + new CfnFirewallConfigReplacementsOutput(accountStack, `FirewallReplacementOutput${instance.instanceName}`, { + instanceId: instance.instanceId, + instanceName: instance.instanceName, + name: firewallConfig.name, + replacements, + }); + } return cluster; } diff --git a/src/lib/cdk-constructs/src/firewall/instance.ts b/src/lib/cdk-constructs/src/firewall/instance.ts index 50dbd20f1..9a005caa5 100644 --- a/src/lib/cdk-constructs/src/firewall/instance.ts +++ b/src/lib/cdk-constructs/src/firewall/instance.ts @@ -50,7 +50,7 @@ export class FirewallInstance extends cdk.Construct { private readonly resource: ec2.CfnInstance; private readonly template: S3Template; private readonly networkInterfacesProps: ec2.CfnInstance.NetworkInterfaceProperty[] = []; - + readonly instanceName: string; constructor(scope: cdk.Construct, id: string, private readonly props: FirewallInstanceProps) { super(scope, id); @@ -110,6 +110,7 @@ export class FirewallInstance extends cdk.Construct { ), }); cdk.Tag.add(this.resource, 'Name', this.props.name); + this.instanceName = this.props.name; this.resource.node.addDependency(this.template); } @@ -204,4 +205,8 @@ export class FirewallInstance extends cdk.Construct { get instanceId() { return this.resource.ref; } + + get replacements(): { [key: string]: string} { + return this.template.replacements; + } } diff --git a/src/lib/common-outputs/src/firewall.ts b/src/lib/common-outputs/src/firewall.ts new file mode 100644 index 000000000..27929d349 --- /dev/null +++ b/src/lib/common-outputs/src/firewall.ts @@ -0,0 +1,17 @@ +import * as t from 'io-ts'; +import { createStructuredOutputFinder } from './structured-output'; + +export const FirewallConfigReplacementsOutput = t.interface( + { + name: t.string, + instanceId: t.string, + instanceName: t.string, + replacements: t.record(t.string, t.string), + }, + 'FirewallConfigReplacementsOutput', +); +export type FirewallConfigReplacementsOutput = t.TypeOf; +export const FirewallConfigReplacementsOutputFinder = createStructuredOutputFinder( + FirewallConfigReplacementsOutput, + () => ({}), +); diff --git a/src/lib/custom-resources/cdk-s3-template/cdk/index.ts b/src/lib/custom-resources/cdk-s3-template/cdk/index.ts index 1d93ccb16..d709bccac 100644 --- a/src/lib/custom-resources/cdk-s3-template/cdk/index.ts +++ b/src/lib/custom-resources/cdk-s3-template/cdk/index.ts @@ -49,6 +49,10 @@ export class S3Template extends cdk.Construct { this.handlerProperties.parameters[key] = value; } + get replacements() { + return this.handlerProperties.parameters; + } + get lambdaFunction(): lambda.Function { return this.ensureLambdaFunction(); } From b153978f2e8519d373a929c49df8e1c6888d3973 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Fri, 25 Sep 2020 10:44:38 +0530 Subject: [PATCH 23/25] Prettier & fixing tests --- .../deployments/firewall/cluster/step-3.ts | 4 ++-- .../test/apps/unsupported-changes.mocks.ts | 24 +++++++++++++++++++ .../cdk-constructs/src/firewall/instance.ts | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts b/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts index b62f2c233..445ad9b63 100644 --- a/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts +++ b/src/deployments/cdk/src/deployments/firewall/cluster/step-3.ts @@ -261,8 +261,8 @@ async function createFirewallCluster(props: { } for (const instance of Object.values(instancePerAz)) { - const replacements: { [key: string]: string} = {}; - Object.entries(instance.replacements).forEach( ([key, value]) => { + const replacements: { [key: string]: string } = {}; + Object.entries(instance.replacements).forEach(([key, value]) => { replacements[key.replace(/[^-a-zA-Z0-9_.]+/gi, '')] = value; }); new CfnFirewallConfigReplacementsOutput(accountStack, `FirewallReplacementOutput${instance.instanceName}`, { diff --git a/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts b/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts index 959a1640a..9832b2f67 100644 --- a/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts +++ b/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts @@ -702,6 +702,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-sharednetwork-phase1-cacentral1-18vq0emthri3h', encryptionKeyArn: 'arn:aws:kms:ca-central-1:007307298200:key/d54a8acb-694c-4fc5-9afe-ca2b263cd0b3', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -932,6 +934,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-operations-phase1-cacentral1-qwupe8qc06ka', encryptionKeyArn: 'arn:aws:kms:ca-central-1:278816265654:key/4e0a5d05-a3ba-4b19-b60e-5f26631d874a', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1014,6 +1018,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-perimeter-phase1-cacentral1-kfs7sxfgn49u', encryptionKeyArn: 'arn:aws:kms:ca-central-1:422986242298:key/ccff8373-96f9-4ced-a167-38476316b235', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1101,6 +1107,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-master-phase1-cacentral1-o4irpt8n8i3p', encryptionKeyArn: 'arn:aws:kms:ca-central-1:687384172140:key/e147a41e-7ada-427f-9b6b-75cdd706e313', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1115,6 +1123,8 @@ export function createPhaseInput(): Omit { bucketArn: 'arn:aws:s3:::pbmmaccel-master-phase0-configcacentral1-3574bod3khwt', bucketName: 'pbmmaccel-master-phase0-configcacentral1-3574bod3khwt', keyPrefix: 'iam-policy', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1129,6 +1139,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-master-phase0-configcacentral1-3574bod3khwt', encryptionKeyArn: 'arn:aws:kms:ca-central-1:687384172140:key/c94a571b-25da-44a1-ac85-366d333ffb2a', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1143,6 +1155,8 @@ export function createPhaseInput(): Omit { bucketArn: 'arn:aws:s3:::pbmmaccel-master-phase0-configcacentral1-3574bod3khwt', bucketName: 'pbmmaccel-master-phase0-configcacentral1-3574bod3khwt', keyPrefix: 'config/scripts/', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1157,6 +1171,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-logarchive-phase0-cacentral1-1fdlszygo5q6l', encryptionKeyArn: 'arn:aws:kms:ca-central-1:272091715658:key/18f7a4af-2fbb-4a4f-a597-7b0bae016c36', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1170,6 +1186,8 @@ export function createPhaseInput(): Omit { bucketArn: 'arn:aws:s3:::pbmmaccel-logarchive-phase0-aescacentral1-7iadcqkmhk3i', bucketName: 'pbmmaccel-logarchive-phase0-aescacentral1-7iadcqkmhk3i', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1184,6 +1202,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-security-phase1-cacentral1-1udpzdaewgqu3', encryptionKeyArn: 'arn:aws:kms:ca-central-1:122259674264:key/ba5d50a0-e25d-4d7e-b15e-bad6d4054310', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1198,6 +1218,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-sharedservices-phase1-cacentral1-1crul1c6woto0', encryptionKeyArn: 'arn:aws:kms:ca-central-1:378053304141:key/f6c1ec02-e1cb-4ace-8abf-25574551cf32', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, @@ -1246,6 +1268,8 @@ export function createPhaseInput(): Omit { bucketName: 'pbmmaccel-funacct-phase1-cacentral1-1qsru3dws5n76', encryptionKeyArn: 'arn:aws:kms:ca-central-1:934027390063:key/7592bb9b-43d1-45d3-be51-bbc59cb06471', region: 'ca-central-1', + encryptionKeyName: 'EncryptionKey', + encryptionKeyId: 'XXXXXXXXXXXXXXXXX', }, }), }, diff --git a/src/lib/cdk-constructs/src/firewall/instance.ts b/src/lib/cdk-constructs/src/firewall/instance.ts index 9a005caa5..264b6f429 100644 --- a/src/lib/cdk-constructs/src/firewall/instance.ts +++ b/src/lib/cdk-constructs/src/firewall/instance.ts @@ -206,7 +206,7 @@ export class FirewallInstance extends cdk.Construct { return this.resource.ref; } - get replacements(): { [key: string]: string} { + get replacements(): { [key: string]: string } { return this.template.replacements; } } From d6d2b8f284041fe433419363b359b1975c69b4e4 Mon Sep 17 00:00:00 2001 From: nachundu Date: Mon, 28 Sep 2020 11:12:30 +0530 Subject: [PATCH 24/25] added ssm access policy --- .../src/save-outputs-to-ssm/iam-utils.ts | 36 +++++++++++++++++++ src/lib/common-outputs/src/iam-role.ts | 7 ++++ 2 files changed, 43 insertions(+) diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts index 52e9a186e..f4d3ec205 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -248,6 +248,42 @@ export async function saveIamPolicy( removalObjects.splice(removalIndex, 1); } } + + const ssmPolicyLength = iamConfig.roles?.filter(r => r['ssm-log-archive-access']).length; + if (ssmPolicyLength && ssmPolicyLength !== 0) { + const ssmPolicyOutput = IamPolicyOutputFinder.findOneByName({ + outputs, + accountKey, + policyKey: 'IamSsmAccessPolicy', + }); + if (!ssmPolicyOutput) { + console.warn(`Didn't find IAM SSM Log Archive Access Policy in output`); + continue; + } + let currentIndex: number; + const previousPolicyIndexDetails = policies.findIndex(p => p.name === ssmPolicyOutput.policyName); + if (previousPolicyIndexDetails >= 0) { + currentIndex = policies[previousPolicyIndexDetails].index; + console.log(`skipping creation of policy ${ssmPolicyOutput.policyName} in SSM`); + } else { + currentIndex = ++policyMaxIndex; + await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/name`, `${ssmPolicyOutput.policyName}`); + await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/arn`, ssmPolicyOutput.policyArn); + policies.push({ + name: ssmPolicyOutput.policyName, + index: currentIndex, + }); + } + updatedPolicies.push({ + index: currentIndex, + name: ssmPolicyOutput.policyName, + }); + + const removalIndex = removalObjects.findIndex(p => p.name === ssmPolicyOutput.policyName); + if (removalIndex !== -1) { + removalObjects.splice(removalIndex, 1); + } + } } for (const removeObject of removalObjects || []) { diff --git a/src/lib/common-outputs/src/iam-role.ts b/src/lib/common-outputs/src/iam-role.ts index 0cb2817f4..15a41963e 100644 --- a/src/lib/common-outputs/src/iam-role.ts +++ b/src/lib/common-outputs/src/iam-role.ts @@ -49,4 +49,11 @@ export const IamPolicyOutputFinder = createStructuredOutputFinder(IamPolicyOutpu accountKey: props.accountKey, predicate: o => o.policyKey === props.policyKey && o.policyName === props.policyName, }), + findOneByName: (props: { outputs: StackOutput[]; accountKey: string; region?: string; policyKey?: string }) => + finder.tryFindOne({ + outputs: props.outputs, + accountKey: props.accountKey, + region: props.region, + predicate: o => o.policyKey === props.policyKey, + }), })); From 4e2def22c3bb2069113f5e98811c3a1f9f9c139e Mon Sep 17 00:00:00 2001 From: nachundu Date: Mon, 28 Sep 2020 12:52:51 +0530 Subject: [PATCH 25/25] fixed prettier issue --- src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts index f4d3ec205..de3ca936a 100644 --- a/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts +++ b/src/core/runtime/src/save-outputs-to-ssm/iam-utils.ts @@ -267,7 +267,10 @@ export async function saveIamPolicy( console.log(`skipping creation of policy ${ssmPolicyOutput.policyName} in SSM`); } else { currentIndex = ++policyMaxIndex; - await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/name`, `${ssmPolicyOutput.policyName}`); + await ssm.putParameter( + `/${acceleratorPrefix}/ident/policy/${currentIndex}/name`, + `${ssmPolicyOutput.policyName}`, + ); await ssm.putParameter(`/${acceleratorPrefix}/ident/policy/${currentIndex}/arn`, ssmPolicyOutput.policyArn); policies.push({ name: ssmPolicyOutput.policyName,