From e9befb4b3b039787f886974d88aa5a4386068964 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 10:38:14 +0530 Subject: [PATCH 1/9] Initial Commit --- src/core/cdk/package.json | 1 + src/core/cdk/src/initial-setup.ts | 12 ++++ src/core/runtime/src/store-outputs.ts | 93 +++++++++++++++++++++++++++ src/lib/common/src/aws/dynamodb.ts | 25 +++++++ 4 files changed, 131 insertions(+) create mode 100644 src/core/runtime/src/store-outputs.ts create mode 100644 src/lib/common/src/aws/dynamodb.ts diff --git a/src/core/cdk/package.json b/src/core/cdk/package.json index 4a35cb3c4..0c526cea1 100644 --- a/src/core/cdk/package.json +++ b/src/core/cdk/package.json @@ -40,6 +40,7 @@ "@aws-cdk/aws-stepfunctions": "1.46.0", "@aws-cdk/aws-stepfunctions-tasks": "1.46.0", "@aws-cdk/aws-secretsmanager": "1.46.0", + "@aws-cdk/aws-dynamodb": "1.46.0", "@aws-cdk/core": "1.46.0", "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 04ed554c6..df5638f32 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -4,6 +4,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as s3assets from '@aws-cdk/aws-s3-assets'; import * as secrets from '@aws-cdk/aws-secretsmanager'; +import * as dynamodb from '@aws-cdk/aws-dynamodb'; import * as sfn from '@aws-cdk/aws-stepfunctions'; import * as tasks from '@aws-cdk/aws-stepfunctions-tasks'; import { CdkDeployProject, PrebuiltCdkDeployProject } from '@aws-accelerator/cdk-accelerator/src/codebuild'; @@ -84,6 +85,17 @@ export namespace InitialSetup { }); setSecretValue(organizationsSecret, '[]'); + const outputsTable = new dynamodb.Table(this, 'Outputs', { + tableName: createName({ + name: 'Outputs', + suffixLength: 0, + }), + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING, + }, + }); + // This is the maximum time before a build times out // The role used by the build should allow this session duration const buildTimeout = cdk.Duration.hours(4); diff --git a/src/core/runtime/src/store-outputs.ts b/src/core/runtime/src/store-outputs.ts new file mode 100644 index 000000000..f8c6ef128 --- /dev/null +++ b/src/core/runtime/src/store-outputs.ts @@ -0,0 +1,93 @@ +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { v4 as uuidv4 } from 'uuid'; + +export interface StoreStackOutputInput { + acceleratorPrefix: string; + assumeRoleName: string; + account: Account; + region: string; + outputsTable: string; +} + +const sts = new STS(); +const dynamodb = new DynamoDB(); + +export const handler = async (input: StoreStackOutputInput) => { + console.log(`Storing stack output...`); + console.log(JSON.stringify(input, null, 2)); + + const { acceleratorPrefix, assumeRoleName, account, region, outputsTable } = input; + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const cfn = new CloudFormation(credentials, region); + const stacks = cfn.listStacksGenerator({ + StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], + }); + const outputs: StackOutput[] = []; + for await (const summary of stacks) { + if (!summary.StackName.startsWith(acceleratorPrefix)) { + console.warn(`Skipping stack with name "${summary.StackName}"`); + continue; + } + const stack = await cfn.describeStack(summary.StackName); + if (!stack) { + console.warn(`Could not load stack with name "${summary.StackName}"`); + continue; + } + const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); + if (!acceleratorTag) { + console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}"`); + continue; + } + + console.debug(`Storing outputs for stack with name "${summary.StackName}"`); + // dynamodb.batchWriteItem({ + // RequestItems: { + + // } + // }) + stack.Outputs?.forEach(output => + outputs.push({ + accountKey: account.key, + outputKey: output.OutputKey, + outputValue: output.OutputValue, + outputDescription: output.Description, + outputExportName: output.ExportName, + region, + }), + ); + } + return { + status: 'SUCCESS', + }; +}; + +const exportOutPutAsDynamoInput = (output: StackOutput) => { + const putRequest = { + Item: { + id: { S: uuidv4() }, + accountKey: { S: output.accountKey }, + region: { S: output.region }, + // outputValue: { ''} + }, + }; +}; + +handler({ + outputsTable: 'PBMMAccel-Outputs', + acceleratorPrefix: 'PBMMAccel-', + assumeRoleName: 'PBMMAccel-PipelineRole', + account: { + key: 'master', + id: '003837753302', + arn: 'arn:aws:organizations::003837753302:account/o-8rceswjlsq/003837753302', + name: 'NonALZ6', + email: 'nkoppula+non-alz-7-master@amazon.com', + ou: 'core', + ouPath: 'core', + }, + region: 'ca-central-1', +}); diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts new file mode 100644 index 000000000..099f8260a --- /dev/null +++ b/src/lib/common/src/aws/dynamodb.ts @@ -0,0 +1,25 @@ +import aws from './aws-client'; +import * as dynamodb from 'aws-sdk/clients/dynamodb'; +import { throttlingBackOff } from './backoff'; + +export class DynamoDB { + private readonly client: aws.DynamoDB; + + constructor(credentials?: aws.Credentials) { + this.client = new aws.DynamoDB({ + credentials, + }); + } + + async createTable(props: dynamodb.CreateTableInput): Promise { + throttlingBackOff(() => this.client.createTable(props).promise()); + } + + async putItem(props: dynamodb.PutItemInput): Promise { + throttlingBackOff(() => this.client.putItem(props).promise()); + } + + async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { + throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + } +} From 74e6212c822d0979dfd3cfa7408f1bc76f8e8b5f Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 16:42:20 +0530 Subject: [PATCH 2/9] Moving outputs from s3 to dynamodb --- src/core/cdk/src/initial-setup.ts | 84 ++++----- .../src/account-default-settings-step.ts | 21 +-- src/core/runtime/src/add-scp-step.ts | 21 +-- .../src/add-tags-to-shared-resources-step.ts | 21 +-- .../src/associate-hosted-zones-step.ts | 20 +- .../runtime/src/create-adconnector/create.ts | 20 +- .../src/create-config-recorder/create.ts | 22 +-- .../src/enable-directory-sharing-step.ts | 21 +-- src/core/runtime/src/store-outputs.ts | 93 ---------- .../runtime/src/store-stack-output-step.ts | 173 ++++++------------ src/core/runtime/src/utils/load-outputs.ts | 19 ++ src/core/runtime/src/verify-files-step.ts | 18 +- src/deployments/cdk/src/utils/outputs.ts | 34 ++-- src/lib/common/src/aws/dynamodb.ts | 14 +- 14 files changed, 194 insertions(+), 387 deletions(-) delete mode 100644 src/core/runtime/src/store-outputs.ts create mode 100644 src/core/runtime/src/utils/load-outputs.ts diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index df5638f32..857fb9a4b 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -481,9 +481,7 @@ export namespace InitialSetup { 'configCommitId.$': '$.configCommitId', 'organizationalUnits.$': '$.organizationalUnits', 'accounts.$': '$.accounts', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -539,12 +537,8 @@ export namespace InitialSetup { ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName, ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName, CONFIG_BRANCH_NAME: props.configBranchName, + STACK_OUTPUT_TABLE_NAME: outputsTable.tableName, }; - if (loadOutputs) { - environment['STACK_OUTPUT_BUCKET_NAME.$'] = '$.storeOutput.outputBucketName'; - environment['STACK_OUTPUT_BUCKET_KEY.$'] = '$.storeOutput.outputBucketKey'; - environment['STACK_OUTPUT_VERSION.$'] = '$.storeOutput.outputVersion'; - } const deployTask = new sfn.Task(this, `Deploy Phase ${phase}`, { // tslint:disable-next-line: deprecation task: new tasks.StartExecution(codeBuildStateMachine, { @@ -559,22 +553,46 @@ export namespace InitialSetup { return deployTask; }; - const createStoreOutputTask = (phase: number) => - new CodeTask(this, `Store Phase ${phase} Output`, { + const createStoreOutputTask = (phase: number) => { + const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs ${phase}`, { + itemsPath: `$.accounts`, + resultPath: 'DISCARD', + maxConcurrency: 1, + parameters: { + 'account.$': '$$.Map.Item.Value', + 'regions.$': '$.regions', + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTable: outputsTable.tableName, + phaseNumber: phase, + } + }); + + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { + itemsPath: `$.regions`, + resultPath: 'DISCARD', + maxConcurrency: 1, + parameters: { + 'account.$': '$.account', + 'region.$': '$$.Map.Item.Value' + }, + }); + + const storeAccountRegionOutputTask = new CodeTask(this, `Store Phase Output ${phase}`, { functionProps: { code: lambdaCode, handler: 'index.storeStackOutputStep', role: pipelineRole, }, - functionPayload: { - acceleratorPrefix: props.acceleratorPrefix, - assumeRoleName: props.stateMachineExecutionRole, - 'accounts.$': '$.accounts', - 'regions.$': '$.regions', - }, - resultPath: '$.storeOutput', + resultPath: 'DISCARD', }); + storeAccountOutputs.iterator(storeAccountRegionOutputs); + storeAccountRegionOutputs.iterator(storeAccountRegionOutputTask); + + return storeAccountOutputs; + } + // TODO Create separate state machine for deployment const deployPhaseRolesTask = createDeploymentTask(-1, false); const storePreviousOutput = createStoreOutputTask(-1); @@ -599,9 +617,7 @@ export namespace InitialSetup { lambdaPath: 'index.createConfigRecorder', name: 'Create Config Recorder', functionPayload: { - 'stackOutputBucketName.$': '$.stackOutputBucketName', - 'stackOutputBucketKey.$': '$.stackOutputBucketKey', - 'stackOutputVersion.$': '$.stackOutputVersion', + outputTableName: outputsTable.tableName, }, }), }); @@ -616,9 +632,7 @@ export namespace InitialSetup { 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', 'baseline.$': '$.baseline', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, acceleratorPrefix: props.acceleratorPrefix, }, }), @@ -638,9 +652,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -660,9 +672,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, rdgwScripts, }, resultPath: 'DISCARD', @@ -680,9 +690,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -695,9 +703,7 @@ export namespace InitialSetup { }, functionPayload: { assumeRoleName: props.stateMachineExecutionRole, - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -714,9 +720,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -740,9 +744,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, }), resultPath: 'DISCARD', diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts index 6e8dc07ff..5f8cc1f5c 100644 --- a/src/core/runtime/src/account-default-settings-step.ts +++ b/src/core/runtime/src/account-default-settings-step.ts @@ -1,6 +1,7 @@ import * as aws from 'aws-sdk'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { StackOutput, getStackOutput, @@ -11,17 +12,15 @@ import { CloudTrail } from '@aws-accelerator/common/src/aws/cloud-trail'; import { PutEventSelectorsRequest, UpdateTrailRequest } from 'aws-sdk/clients/cloudtrail'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { loadOutputs } from './utils/load-outputs'; interface AccountDefaultSettingsInput extends LoadConfigurationInput { assumeRoleName: string; accounts: Account[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: AccountDefaultSettingsInput) => { console.log('Setting account level defaults for all accounts in an organization ...'); @@ -33,17 +32,9 @@ export const handler = async (input: AccountDefaultSettingsInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ repositoryName: configRepositoryName, @@ -53,7 +44,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => { const logAccountKey = acceleratorConfig.getMandatoryAccountKey('central-log'); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index 906420aec..ba018ad57 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -2,21 +2,19 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts'; import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; +import { loadOutputs } from './utils/load-outputs'; interface AddScpInput extends LoadConfigurationInput { acceleratorPrefix: string; accounts: Account[]; organizationalUnits: OrganizationalUnit[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: AddScpInput) => { console.log(`Adding service control policy to organization...`); @@ -29,9 +27,7 @@ export const handler = async (input: AddScpInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -43,12 +39,7 @@ export const handler = async (input: AddScpInput) => { const organizationAdminRole = config['global-options']['organization-admin-role']!; const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // Find the SCP artifact output const artifactOutput = ArtifactOutputFinder.findOneByName({ diff --git a/src/core/runtime/src/add-tags-to-shared-resources-step.ts b/src/core/runtime/src/add-tags-to-shared-resources-step.ts index a2c195ca7..5473b8889 100644 --- a/src/core/runtime/src/add-tags-to-shared-resources-step.ts +++ b/src/core/runtime/src/add-tags-to-shared-resources-step.ts @@ -1,15 +1,14 @@ import { TagResources } from '@aws-accelerator/common/src/aws/resource-tagging'; import { STS } from '@aws-accelerator/common/src/aws/sts'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { loadOutputs } from './utils/load-outputs'; const ALLOWED_RESOURCE_TYPES = ['subnet', 'security-group', 'vpc', 'tgw-attachment']; interface CreateTagsRequestInput { assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface Tag { @@ -26,22 +25,16 @@ interface AddTagToResourceOutput { type AddTagToResourceOutputs = AddTagToResourceOutput[]; -const s3 = new S3(); +const dynamodb = new DynamoDB(); const sts = new STS(); export const handler = async (input: CreateTagsRequestInput) => { console.log(`Adding tags to shared resource...`); console.log(JSON.stringify(input, null, 2)); - const { assumeRoleName, stackOutputBucketName, stackOutputBucketKey, stackOutputVersion } = input; - - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const { assumeRoleName, outputTableName } = input; + const outputs = await loadOutputs(outputTableName, dynamodb); const addTagsToResourcesOutputs: AddTagToResourceOutputs[] = getStackJsonOutput(outputs, { outputType: 'AddTagsToResources', }); diff --git a/src/core/runtime/src/associate-hosted-zones-step.ts b/src/core/runtime/src/associate-hosted-zones-step.ts index 9a06b89b6..08b1705b7 100644 --- a/src/core/runtime/src/associate-hosted-zones-step.ts +++ b/src/core/runtime/src/associate-hosted-zones-step.ts @@ -1,5 +1,5 @@ import * as r53 from 'aws-sdk/clients/route53'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { getStackJsonOutput, StackOutput, ResolversOutput } from '@aws-accelerator/common-outputs/src/stack-output'; @@ -9,13 +9,12 @@ import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { throttlingBackOff } from '@aws-accelerator/common/src/aws/backoff'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; +import { loadOutputs } from './utils/load-outputs'; interface AssociateHostedZonesInput extends LoadConfigurationInput { accounts: Account[]; assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } type ResolversOutputs = ResolversOutput[]; @@ -42,7 +41,7 @@ interface AccountRule { // Hosted zone ID is in the form of `/hostedzone/Z0181099DGX53XMU1D7S` const hostedZoneIdRegex = /\/hostedzone\/([\d\w]+)/; -const s3 = new S3(); +const dynamodb = new DynamoDB(); const sts = new STS(); export const handler = async (input: AssociateHostedZonesInput) => { @@ -55,9 +54,7 @@ export const handler = async (input: AssociateHostedZonesInput) => { assumeRoleName, configCommitId, configFilePath, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -67,12 +64,7 @@ export const handler = async (input: AssociateHostedZonesInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // get the private zones from global-options const globalOptionsConfig = config['global-options']; diff --git a/src/core/runtime/src/create-adconnector/create.ts b/src/core/runtime/src/create-adconnector/create.ts index d1d8b8632..710ecb288 100644 --- a/src/core/runtime/src/create-adconnector/create.ts +++ b/src/core/runtime/src/create-adconnector/create.ts @@ -1,6 +1,6 @@ import { DirectoryService } from '@aws-accelerator/common/src/aws/directory-service'; import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { createMadUserPasswordSecretName, MadOutput } from '@aws-accelerator/common-outputs/src/mad'; @@ -8,6 +8,7 @@ import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; +import { loadOutputs } from '../utils/load-outputs'; const VALID_STATUSES: string[] = ['Requested', 'Creating', 'Created', 'Active', 'Inoperable', 'Impaired', 'Restoring']; @@ -18,9 +19,7 @@ interface AdConnectorInput extends LoadConfigurationInput { configRepositoryName: string; configFilePath: string; configCommitId: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } export interface AdConnectorOutput { @@ -29,7 +28,7 @@ export interface AdConnectorOutput { assumeRoleName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); const secrets = new SecretsManager(); const sts = new STS(); @@ -44,9 +43,7 @@ export const handler = async (input: AdConnectorInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -56,12 +53,7 @@ export const handler = async (input: AdConnectorInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); const adConnectorOutputs: AdConnectorOutput[] = []; for (const [accountKey, mandatoryConfig] of acceleratorConfig.getMandatoryAccountConfigs()) { diff --git a/src/core/runtime/src/create-config-recorder/create.ts b/src/core/runtime/src/create-config-recorder/create.ts index f20e3a8d3..761ed709c 100644 --- a/src/core/runtime/src/create-config-recorder/create.ts +++ b/src/core/runtime/src/create-config-recorder/create.ts @@ -1,21 +1,20 @@ import { ConfigService } from '@aws-accelerator/common/src/aws/configservice'; import { ConfigurationRecorder } from 'aws-sdk/clients/configservice'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { createConfigRecorderName, createAggregatorName } from '@aws-accelerator/common-outputs/src/config'; import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; +import { loadOutputs } from '../utils/load-outputs'; interface ConfigServiceInput extends LoadConfigurationInput { account: Account; assumeRoleName: string; acceleratorPrefix: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface LogBucketOutputType { @@ -36,7 +35,7 @@ const CustomErrorMessage = [ ]; const sts = new STS(); -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: ConfigServiceInput): Promise => { console.log(`Enable Config Recorder in account ...`); @@ -48,17 +47,10 @@ export const handler = async (input: ConfigServiceInput): Promise => { configFilePath, configCommitId, acceleratorPrefix, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/enable-directory-sharing-step.ts b/src/core/runtime/src/enable-directory-sharing-step.ts index b7c19da9c..66e83a08a 100644 --- a/src/core/runtime/src/enable-directory-sharing-step.ts +++ b/src/core/runtime/src/enable-directory-sharing-step.ts @@ -1,21 +1,20 @@ import { DirectoryService } from '@aws-accelerator/common/src/aws/directory-service'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { MadOutput } from '@aws-accelerator/common-outputs/src/mad'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; +import { loadOutputs } from './utils/load-outputs'; interface ShareDirectoryInput extends LoadConfigurationInput { accounts: Account[]; assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: ShareDirectoryInput) => { console.log(`Sharing MAD to another account ...`); @@ -27,9 +26,7 @@ export const handler = async (input: ShareDirectoryInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -39,12 +36,8 @@ export const handler = async (input: ShareDirectoryInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + + const outputs = await loadOutputs(outputTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/store-outputs.ts b/src/core/runtime/src/store-outputs.ts deleted file mode 100644 index f8c6ef128..000000000 --- a/src/core/runtime/src/store-outputs.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -import { STS } from '@aws-accelerator/common/src/aws/sts'; -import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { v4 as uuidv4 } from 'uuid'; - -export interface StoreStackOutputInput { - acceleratorPrefix: string; - assumeRoleName: string; - account: Account; - region: string; - outputsTable: string; -} - -const sts = new STS(); -const dynamodb = new DynamoDB(); - -export const handler = async (input: StoreStackOutputInput) => { - console.log(`Storing stack output...`); - console.log(JSON.stringify(input, null, 2)); - - const { acceleratorPrefix, assumeRoleName, account, region, outputsTable } = input; - const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); - const cfn = new CloudFormation(credentials, region); - const stacks = cfn.listStacksGenerator({ - StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], - }); - const outputs: StackOutput[] = []; - for await (const summary of stacks) { - if (!summary.StackName.startsWith(acceleratorPrefix)) { - console.warn(`Skipping stack with name "${summary.StackName}"`); - continue; - } - const stack = await cfn.describeStack(summary.StackName); - if (!stack) { - console.warn(`Could not load stack with name "${summary.StackName}"`); - continue; - } - const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); - if (!acceleratorTag) { - console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}"`); - continue; - } - - console.debug(`Storing outputs for stack with name "${summary.StackName}"`); - // dynamodb.batchWriteItem({ - // RequestItems: { - - // } - // }) - stack.Outputs?.forEach(output => - outputs.push({ - accountKey: account.key, - outputKey: output.OutputKey, - outputValue: output.OutputValue, - outputDescription: output.Description, - outputExportName: output.ExportName, - region, - }), - ); - } - return { - status: 'SUCCESS', - }; -}; - -const exportOutPutAsDynamoInput = (output: StackOutput) => { - const putRequest = { - Item: { - id: { S: uuidv4() }, - accountKey: { S: output.accountKey }, - region: { S: output.region }, - // outputValue: { ''} - }, - }; -}; - -handler({ - outputsTable: 'PBMMAccel-Outputs', - acceleratorPrefix: 'PBMMAccel-', - assumeRoleName: 'PBMMAccel-PipelineRole', - account: { - key: 'master', - id: '003837753302', - arn: 'arn:aws:organizations::003837753302:account/o-8rceswjlsq/003837753302', - name: 'NonALZ6', - email: 'nkoppula+non-alz-7-master@amazon.com', - ou: 'core', - ouPath: 'core', - }, - region: 'ca-central-1', -}); diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index 71636db54..1abf7e454 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -1,144 +1,83 @@ -import * as aws from 'aws-sdk'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { collectAsync } from '@aws-accelerator/common/src/util/generator'; -import { CentralBucketOutputFinder } from '@aws-accelerator/common-outputs/src/central-bucket'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; export interface StoreStackOutputInput { acceleratorPrefix: string; assumeRoleName: string; - accounts: Account[]; - regions: string[]; + account: Account; + region: string; + outputsTable: string; + phaseNumber: number; } -const s3 = new S3(); const sts = new STS(); +const dynamodb = new DynamoDB(); export const handler = async (input: StoreStackOutputInput) => { console.log(`Storing stack output...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, assumeRoleName, accounts, regions } = input; - - const outputsAsyncIterable = getOutputsForAccountsAndRegions({ - acceleratorPrefix, - accounts, - assumeRoleName, - regions, - }); - const outputs = await collectAsync(outputsAsyncIterable); - - // Find the central output bucket in outputs - const centralBucketOutput = CentralBucketOutputFinder.tryFindOne({ - outputs, + const { acceleratorPrefix, assumeRoleName, account, region, outputsTable, phaseNumber } = input; + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const cfn = new CloudFormation(credentials, region); + const stacks = cfn.listStacksGenerator({ + StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], }); + const outputs: StackOutput[] = []; + for await (const summary of stacks) { + if (!summary.StackName.match(`${acceleratorPrefix}(.*)-Phase${phaseNumber}`)) { + console.warn(`Skipping stack with name "${summary.StackName}"`); + continue; + } + const stack = await cfn.describeStack(summary.StackName); + if (!stack) { + console.warn(`Could not load stack with name "${summary.StackName}"`); + continue; + } + const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); + if (!acceleratorTag) { + console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}"`); + continue; + } - if (!centralBucketOutput) { - console.log(`Didn't find "centralBucket" in existing outputs, might be first run of Accelerator`); + console.debug(`Storing outputs for stack with name "${summary.StackName}"`); + stack.Outputs?.forEach(output => + outputs.push({ + accountKey: account.key, + outputKey: `${output.OutputKey}sjkdh`, + outputValue: output.OutputValue, + outputDescription: output.Description, + outputExportName: output.ExportName, + region, + }), + ); + } + if (outputs.length === 0) { + console.warn(`No outputs found for Account: ${account.key} and Region: ${region}`); + await dynamodb.deleteItem({ + TableName: outputsTable, + Key: { + id: { S: `${account.key}-${region}-${phaseNumber}` }, + }, + }); return { status: 'SUCCESS', - outputBucketName: '', - outputBucketKey: '', - outputVersion: '', }; } - - console.log(`Writing outputs to s3://${centralBucketOutput.bucketName}/outputs.json`); - - // Store outputs on S3 - const response = await s3.putObject({ - Bucket: centralBucketOutput.bucketName, - Key: 'outputs.json', - Body: JSON.stringify(outputs), + await dynamodb.putItem({ + Item: { + id: { S: `${account.key}-${region}-${phaseNumber}` }, + accountKey: { S: account.key }, + region: { S: region }, + phase: { N: `${phaseNumber}` }, + outputValue: { S: JSON.stringify(outputs) }, + }, + TableName: outputsTable, }); - return { status: 'SUCCESS', - outputBucketName: centralBucketOutput.bucketName, - outputBucketKey: 'outputs.json', - outputVersion: response.VersionId, }; }; - -/** - * Find all outputs in the given accounts and regions. - */ -async function* getOutputsForAccountsAndRegions(props: { - acceleratorPrefix: string; - accounts: Account[]; - assumeRoleName: string; - regions: string[]; -}): AsyncIterableIterator { - const { acceleratorPrefix, accounts, assumeRoleName, regions } = props; - const outputsListPromises = []; - for (const account of accounts) { - const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); - for (const region of regions) { - const outputsListPromise = getOutputsForRegion({ - acceleratorPrefix, - accountKey: account.key, - credentials, - region, - }); - outputsListPromises.push(outputsListPromise); - } - } - for (const outputsList of outputsListPromises) { - yield* outputsList; - } -} - -/** - * Find all outputs in the given account and region. - */ -async function* getOutputsForRegion(props: { - acceleratorPrefix: string; - accountKey: string; - credentials: aws.Credentials; - region: string; -}): AsyncIterableIterator { - const { acceleratorPrefix, accountKey, credentials, region } = props; - - try { - const cfn = new CloudFormation(credentials, region); - - const stacks = cfn.listStacksGenerator({ - StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], - }); - - for await (const summary of stacks) { - if (!summary.StackName.startsWith(acceleratorPrefix)) { - console.warn(`Skipping stack with name "${summary.StackName}"`); - continue; - } - const stack = await cfn.describeStack(summary.StackName); - if (!stack) { - console.warn(`Could not load stack with name "${summary.StackName}"`); - continue; - } - const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); - if (!acceleratorTag) { - console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}" in region ${region}`); - continue; - } - - console.debug(`Storing outputs for stack with name "${summary.StackName}" in region ${region}`); - for (const output of stack.Outputs || []) { - yield { - accountKey, - region, - outputKey: output.OutputKey, - outputValue: output.OutputValue, - outputDescription: output.Description, - outputExportName: output.ExportName, - }; - } - } - } catch (e) { - console.warn(`Cannot find outputs in account "${accountKey}" and region "${region}": ${e}`); - } -} diff --git a/src/core/runtime/src/utils/load-outputs.ts b/src/core/runtime/src/utils/load-outputs.ts new file mode 100644 index 000000000..4e5003728 --- /dev/null +++ b/src/core/runtime/src/utils/load-outputs.ts @@ -0,0 +1,19 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; + + +export async function loadOutputs(tableName: string, client: DynamoDB): Promise { + const outputs: StackOutput[] = []; + const outputsResponse = await client.scan({ + TableName: tableName, + }); + if (!outputsResponse.Items) { + console.warn(`Did not find outputs in DynamoDB table "${tableName}"`); + return []; + } + for (const item of outputsResponse.Items) { + const cVal = JSON.parse(item.outputValue.S!); + outputs.push(...cVal); + } + return outputs; +} \ No newline at end of file diff --git a/src/core/runtime/src/verify-files-step.ts b/src/core/runtime/src/verify-files-step.ts index 4f02bfc03..3366f5640 100644 --- a/src/core/runtime/src/verify-files-step.ts +++ b/src/core/runtime/src/verify-files-step.ts @@ -1,16 +1,16 @@ import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts'; import { CentralBucketOutputFinder } from '@aws-accelerator/common-outputs/src/central-bucket'; import * as c from '@aws-accelerator/common-config/src'; +import { loadOutputs } from './utils/load-outputs'; interface VerifyFilesInput extends LoadConfigurationInput { rdgwScripts: string[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface RdgwArtifactsOutput { @@ -21,6 +21,7 @@ interface RdgwArtifactsOutput { } const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: VerifyFilesInput) => { console.log('Validate existence of all required files ...'); @@ -31,17 +32,10 @@ export const handler = async (input: VerifyFilesInput) => { configFilePath, configCommitId, rdgwScripts, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/deployments/cdk/src/utils/outputs.ts b/src/deployments/cdk/src/utils/outputs.ts index c0e81dae0..ee4739948 100644 --- a/src/deployments/cdk/src/utils/outputs.ts +++ b/src/deployments/cdk/src/utils/outputs.ts @@ -1,9 +1,9 @@ import * as fs from 'fs'; import * as path from 'path'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -const s3 = new S3(); +const dynamodb = new DynamoDB(); export async function loadStackOutputs(): Promise { if (process.env.CONFIG_MODE === 'development') { @@ -15,29 +15,23 @@ export async function loadStackOutputs(): Promise { return JSON.parse(contents.toString()); } - const outputBucketName = process.env.STACK_OUTPUT_BUCKET_NAME; - const outputBucketKey = process.env.STACK_OUTPUT_BUCKET_KEY; - const outputVersion = process.env.STACK_OUTPUT_VERSION; - if (!outputBucketName || !outputBucketKey || !outputVersion) { - console.warn( - `The environment variable "STACK_OUTPUT_BUCKET_NAME", "STACK_OUTPUT_BUCKET_KEY", "STACK_OUTPUT_VERSION" need to be set`, - ); + const outputTableName = process.env.STACK_OUTPUT_TABLE_NAME; + if (!outputTableName) { + console.warn(`The environment variable "STACK_OUTPUT_TABLE_NAME" need to be set`); return []; } - const outputsJson = await s3.getObjectBodyAsString({ - Bucket: outputBucketName, - Key: outputBucketKey, - VersionId: outputVersion, + const outputs: StackOutput[] = []; + const outputsResponse = await dynamodb.scan({ + TableName: outputTableName, }); - if (!outputsJson) { - console.warn(`Cannot find outputs "s3://${outputBucketName}${outputBucketKey}"`); + if (!outputsResponse.Items) { + console.warn(`Did not find outputs in DynamoDB table "${outputTableName}"`); return []; } - try { - return JSON.parse(outputsJson); - } catch (e) { - console.warn(`Cannot parse outputs "s3://${outputBucketName}${outputBucketKey}"`); - return []; + for (const item of outputsResponse.Items) { + const cVal = JSON.parse(item.outputValue.S!); + outputs.push(...cVal); } + return outputs; } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 099f8260a..87325c0fe 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -12,14 +12,22 @@ export class DynamoDB { } async createTable(props: dynamodb.CreateTableInput): Promise { - throttlingBackOff(() => this.client.createTable(props).promise()); + await throttlingBackOff(() => this.client.createTable(props).promise()); } async putItem(props: dynamodb.PutItemInput): Promise { - throttlingBackOff(() => this.client.putItem(props).promise()); + await throttlingBackOff(() => this.client.putItem(props).promise()); } async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { - throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + } + + async scan(props: dynamodb.ScanInput): Promise { + return throttlingBackOff(() => this.client.scan(props).promise()); + } + + async deleteItem(props: dynamodb.DeleteItemInput): Promise { + await throttlingBackOff(() => this.client.deleteItem(props).promise()); } } From 38a180e741d3b57780873b9f2f8341bcdc0b01d6 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 16:42:26 +0530 Subject: [PATCH 3/9] Prettier --- src/core/cdk/src/initial-setup.ts | 8 ++--- .../src/account-default-settings-step.ts | 9 +----- .../src/associate-hosted-zones-step.ts | 9 +----- .../src/enable-directory-sharing-step.ts | 10 +------ src/core/runtime/src/utils/load-outputs.ts | 29 +++++++++---------- src/core/runtime/src/verify-files-step.ts | 8 +---- 6 files changed, 22 insertions(+), 51 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 857fb9a4b..334cd354e 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -565,16 +565,16 @@ export namespace InitialSetup { assumeRoleName: props.stateMachineExecutionRole, outputsTable: outputsTable.tableName, phaseNumber: phase, - } + }, }); - + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { itemsPath: `$.regions`, resultPath: 'DISCARD', maxConcurrency: 1, parameters: { 'account.$': '$.account', - 'region.$': '$$.Map.Item.Value' + 'region.$': '$$.Map.Item.Value', }, }); @@ -591,7 +591,7 @@ export namespace InitialSetup { storeAccountRegionOutputs.iterator(storeAccountRegionOutputTask); return storeAccountOutputs; - } + }; // TODO Create separate state machine for deployment const deployPhaseRolesTask = createDeploymentTask(-1, false); diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts index 5f8cc1f5c..1ef165997 100644 --- a/src/core/runtime/src/account-default-settings-step.ts +++ b/src/core/runtime/src/account-default-settings-step.ts @@ -26,14 +26,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => { console.log('Setting account level defaults for all accounts in an organization ...'); console.log(JSON.stringify(input, null, 2)); - const { - assumeRoleName, - accounts, - configRepositoryName, - configFilePath, - configCommitId, - outputTableName, - } = input; + const { assumeRoleName, accounts, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/associate-hosted-zones-step.ts b/src/core/runtime/src/associate-hosted-zones-step.ts index 08b1705b7..8f4fa104f 100644 --- a/src/core/runtime/src/associate-hosted-zones-step.ts +++ b/src/core/runtime/src/associate-hosted-zones-step.ts @@ -48,14 +48,7 @@ export const handler = async (input: AssociateHostedZonesInput) => { console.log(`Associating Hosted Zones with VPC...`); console.log(JSON.stringify(input, null, 2)); - const { - configRepositoryName, - accounts, - assumeRoleName, - configCommitId, - configFilePath, - outputTableName, - } = input; + const { configRepositoryName, accounts, assumeRoleName, configCommitId, configFilePath, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/enable-directory-sharing-step.ts b/src/core/runtime/src/enable-directory-sharing-step.ts index 66e83a08a..b7d9c030b 100644 --- a/src/core/runtime/src/enable-directory-sharing-step.ts +++ b/src/core/runtime/src/enable-directory-sharing-step.ts @@ -20,14 +20,7 @@ export const handler = async (input: ShareDirectoryInput) => { console.log(`Sharing MAD to another account ...`); console.log(JSON.stringify(input, null, 2)); - const { - accounts, - assumeRoleName, - configRepositoryName, - configFilePath, - configCommitId, - outputTableName, - } = input; + const { accounts, assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ @@ -36,7 +29,6 @@ export const handler = async (input: ShareDirectoryInput) => { commitId: configCommitId, }); - const outputs = await loadOutputs(outputTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/utils/load-outputs.ts b/src/core/runtime/src/utils/load-outputs.ts index 4e5003728..83850d0ae 100644 --- a/src/core/runtime/src/utils/load-outputs.ts +++ b/src/core/runtime/src/utils/load-outputs.ts @@ -1,19 +1,18 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; - export async function loadOutputs(tableName: string, client: DynamoDB): Promise { - const outputs: StackOutput[] = []; - const outputsResponse = await client.scan({ - TableName: tableName, - }); - if (!outputsResponse.Items) { - console.warn(`Did not find outputs in DynamoDB table "${tableName}"`); - return []; - } - for (const item of outputsResponse.Items) { - const cVal = JSON.parse(item.outputValue.S!); - outputs.push(...cVal); - } - return outputs; -} \ No newline at end of file + const outputs: StackOutput[] = []; + const outputsResponse = await client.scan({ + TableName: tableName, + }); + if (!outputsResponse.Items) { + console.warn(`Did not find outputs in DynamoDB table "${tableName}"`); + return []; + } + for (const item of outputsResponse.Items) { + const cVal = JSON.parse(item.outputValue.S!); + outputs.push(...cVal); + } + return outputs; +} diff --git a/src/core/runtime/src/verify-files-step.ts b/src/core/runtime/src/verify-files-step.ts index 3366f5640..c500d2ffa 100644 --- a/src/core/runtime/src/verify-files-step.ts +++ b/src/core/runtime/src/verify-files-step.ts @@ -27,13 +27,7 @@ export const handler = async (input: VerifyFilesInput) => { console.log('Validate existence of all required files ...'); console.log(JSON.stringify(input, null, 2)); - const { - configRepositoryName, - configFilePath, - configCommitId, - rdgwScripts, - outputTableName, - } = input; + const { configRepositoryName, configFilePath, configCommitId, rdgwScripts, outputTableName } = input; const outputs = await loadOutputs(outputTableName, dynamodb); From 132d313c86d102322fdb957577f2227fefb91d49 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 17:32:48 +0530 Subject: [PATCH 4/9] Sending payload to store outputs --- src/core/cdk/src/initial-setup.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 334cd354e..6f9b0bf5b 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -561,10 +561,6 @@ export namespace InitialSetup { parameters: { 'account.$': '$$.Map.Item.Value', 'regions.$': '$.regions', - acceleratorPrefix: props.acceleratorPrefix, - assumeRoleName: props.stateMachineExecutionRole, - outputsTable: outputsTable.tableName, - phaseNumber: phase, }, }); @@ -584,6 +580,12 @@ export namespace InitialSetup { handler: 'index.storeStackOutputStep', role: pipelineRole, }, + functionPayload: { + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTable: outputsTable.tableName, + phaseNumber: phase, + }, resultPath: 'DISCARD', }); From 21c40915e90ddc92f0ad8af34fa86c858c34089a Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 18:30:16 +0530 Subject: [PATCH 5/9] sending params to map sm --- src/core/cdk/src/initial-setup.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 6f9b0bf5b..02a377910 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -561,6 +561,10 @@ export namespace InitialSetup { parameters: { 'account.$': '$$.Map.Item.Value', 'regions.$': '$.regions', + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTable: outputsTable.tableName, + phaseNumber: phase, }, }); @@ -571,6 +575,10 @@ export namespace InitialSetup { parameters: { 'account.$': '$.account', 'region.$': '$$.Map.Item.Value', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', }, }); @@ -580,12 +588,6 @@ export namespace InitialSetup { handler: 'index.storeStackOutputStep', role: pipelineRole, }, - functionPayload: { - acceleratorPrefix: props.acceleratorPrefix, - assumeRoleName: props.stateMachineExecutionRole, - outputsTable: outputsTable.tableName, - phaseNumber: phase, - }, resultPath: 'DISCARD', }); From ce0de7029b6528dcae70bded9a7c59938cc8ded6 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 31 Aug 2020 18:59:05 +0530 Subject: [PATCH 6/9] Incresing maxConcurrency to 10 for accounts and regions --- src/core/cdk/src/initial-setup.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 02a377910..334d43a9a 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -557,7 +557,7 @@ export namespace InitialSetup { const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs ${phase}`, { itemsPath: `$.accounts`, resultPath: 'DISCARD', - maxConcurrency: 1, + maxConcurrency: 10, parameters: { 'account.$': '$$.Map.Item.Value', 'regions.$': '$.regions', @@ -571,7 +571,7 @@ export namespace InitialSetup { const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { itemsPath: `$.regions`, resultPath: 'DISCARD', - maxConcurrency: 1, + maxConcurrency: 10, parameters: { 'account.$': '$.account', 'region.$': '$$.Map.Item.Value', From 12dd12fe35afc055515f702d62142f81f8769e88 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 1 Sep 2020 11:06:40 +0530 Subject: [PATCH 7/9] Interm Push --- src/core/cdk/src/initial-setup.ts | 129 ++++++++++++------- src/core/cdk/src/tasks/store-outputs-task.ts | 77 +++++++++++ 2 files changed, 162 insertions(+), 44 deletions(-) create mode 100644 src/core/cdk/src/tasks/store-outputs-task.ts diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 334d43a9a..375fb01cf 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -20,6 +20,7 @@ import { CreateStackTask } from './tasks/create-stack-task'; import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task'; import * as fs from 'fs'; import * as sns from '@aws-cdk/aws-sns'; +import { StoreOutputsTask } from './tasks/store-outputs-task'; export namespace InitialSetup { export interface CommonProps { @@ -553,52 +554,92 @@ export namespace InitialSetup { return deployTask; }; - const createStoreOutputTask = (phase: number) => { - const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs ${phase}`, { - itemsPath: `$.accounts`, - resultPath: 'DISCARD', - maxConcurrency: 10, - parameters: { - 'account.$': '$$.Map.Item.Value', - 'regions.$': '$.regions', - acceleratorPrefix: props.acceleratorPrefix, - assumeRoleName: props.stateMachineExecutionRole, - outputsTable: outputsTable.tableName, - phaseNumber: phase, - }, - }); - const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { - itemsPath: `$.regions`, - resultPath: 'DISCARD', - maxConcurrency: 10, - parameters: { - 'account.$': '$.account', - 'region.$': '$$.Map.Item.Value', - 'acceleratorPrefix.$': '$.acceleratorPrefix', - 'assumeRoleName.$': '$.assumeRoleName', - 'outputsTable.$': '$.outputsTable', - 'phaseNumber.$': '$.phaseNumber', - }, - }); + const storeOutputsStateMachine = new sfn.StateMachine(this, `${props.acceleratorPrefix}StoreOutputs_sm`, { + stateMachineName: `${props.acceleratorPrefix}StoreOutputs_sm`, + definition: new StoreOutputsTask(this, 'StoreOutputs', { + lambdaCode, + role: pipelineRole, + }), + }); - const storeAccountRegionOutputTask = new CodeTask(this, `Store Phase Output ${phase}`, { - functionProps: { - code: lambdaCode, - handler: 'index.storeStackOutputStep', - role: pipelineRole, - }, + const createStoreOutputTask = (phase: number) => { + + const environment: { [name: string]: string } = { + ACCELERATOR_PHASE: `${phase}`, + 'CONFIG_REPOSITORY_NAME.$': '$.configRepositoryName', + 'CONFIG_FILE_PATH.$': '$.configFilePath', + 'CONFIG_COMMIT_ID.$': '$.configCommitId', + 'ACCELERATOR_BASELINE.$': '$.baseline', + 'CONFIG_ROOT_FILE_PATH.$': '$.configRootFilePath', + ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName, + ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName, + CONFIG_BRANCH_NAME: props.configBranchName, + STACK_OUTPUT_TABLE_NAME: outputsTable.tableName, + }; + const storeOutputsTask = new sfn.Task(this, `Store Phase ${phase} Outputs`, { + // tslint:disable-next-line: deprecation + task: new tasks.StartExecution(storeOutputsStateMachine, { + integrationPattern: sfn.ServiceIntegrationPattern.SYNC, + input: { + 'accounts.$': '$.accounts', + 'regions.$': '$.regions', + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTable: outputsTable.tableName, + phaseNumber: phase, + }, + }), resultPath: 'DISCARD', }); - - storeAccountOutputs.iterator(storeAccountRegionOutputs); - storeAccountRegionOutputs.iterator(storeAccountRegionOutputTask); - - return storeAccountOutputs; + return storeOutputsTask; + + + // const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs ${phase}`, { + // itemsPath: `$.accounts`, + // resultPath: 'DISCARD', + // maxConcurrency: 10, + // parameters: { + // 'account.$': '$$.Map.Item.Value', + // 'regions.$': '$.regions', + // acceleratorPrefix: props.acceleratorPrefix, + // assumeRoleName: props.stateMachineExecutionRole, + // outputsTable: outputsTable.tableName, + // phaseNumber: phase, + // }, + // }); + + // const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { + // itemsPath: `$.regions`, + // resultPath: 'DISCARD', + // maxConcurrency: 10, + // parameters: { + // 'account.$': '$.account', + // 'region.$': '$$.Map.Item.Value', + // 'acceleratorPrefix.$': '$.acceleratorPrefix', + // 'assumeRoleName.$': '$.assumeRoleName', + // 'outputsTable.$': '$.outputsTable', + // 'phaseNumber.$': '$.phaseNumber', + // }, + // }); + + // const storeAccountRegionOutputTask = new CodeTask(this, `Store Phase Output ${phase}`, { + // functionProps: { + // code: lambdaCode, + // handler: 'index.storeStackOutputStep', + // role: pipelineRole, + // }, + // resultPath: 'DISCARD', + // }); + + // storeAccountOutputs.iterator(storeAccountRegionOutputs); + // storeAccountRegionOutputs.iterator(storeAccountRegionOutputTask); + + // return storeAccountOutputs; }; // TODO Create separate state machine for deployment - const deployPhaseRolesTask = createDeploymentTask(-1, false); + // const deployPhaseRolesTask = createDeploymentTask(-1, false); const storePreviousOutput = createStoreOutputTask(-1); const deployPhase0Task = createDeploymentTask(0); const storePhase0Output = createStoreOutputTask(0); @@ -797,11 +838,11 @@ export namespace InitialSetup { const commonDefinition = loadOrganizationsTask.startState .next(loadAccountsTask) - .next(installRolesTask) - .next(deleteVpcTask) - .next(loadLimitsTask) - .next(enableTrustedAccessForServicesTask) - .next(deployPhaseRolesTask) + // .next(installRolesTask) + // .next(deleteVpcTask) + // .next(loadLimitsTask) + // .next(enableTrustedAccessForServicesTask) + // .next(deployPhaseRolesTask) .next(storePreviousOutput) .next(deployPhase0Task) .next(storePhase0Output) diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts new file mode 100644 index 000000000..790a7356f --- /dev/null +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -0,0 +1,77 @@ +import * as cdk from '@aws-cdk/core'; +import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { CodeTask } from '@aws-accelerator/cdk-accelerator/src/stepfunction-tasks'; + +export namespace StoreOutputsTask { + export interface Props { + role: iam.IRole; + lambdaCode: lambda.Code; + functionPayload?: { [key: string]: unknown }; + waitSeconds?: number; + } +} + +export class StoreOutputsTask extends sfn.StateMachineFragment { + readonly startState: sfn.State; + readonly endStates: sfn.INextable[]; + + constructor(scope: cdk.Construct, id: string, props: StoreOutputsTask.Props) { + super(scope, id); + + const { role, lambdaCode, functionPayload, waitSeconds = 10 } = props; + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + resources: ['*'], + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + }), + ); + + const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs`, { + itemsPath: `$.accounts`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$$.Map.Item.Value', + 'regions.$': '$.regions', + ...functionPayload, + }, + }); + + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, { + itemsPath: `$.regions`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$.account', + 'region.$': '$$.Map.Item.Value', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', + }, + }); + + const startTaskResultPath = '$.storeOutputsOutput'; + const storeOutputsTask = new CodeTask(scope, `Store Outputs`, { + resultPath: startTaskResultPath, + functionPayload, + functionProps: { + role, + code: lambdaCode, + handler: 'index.storeStackOutputStep', + }, + }); + + const pass = new sfn.Pass(this, 'Store Outputs Success'); + storeAccountOutputs.iterator(storeAccountRegionOutputs); + storeAccountRegionOutputs.iterator(storeOutputsTask); + const chain = sfn.Chain.start(storeAccountOutputs).next(pass) + + this.startState = chain.startState; + this.endStates = chain.endStates; + } +} From bce57e1e312eec9162de9485a067d0c7e4961b05 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 1 Sep 2020 14:29:19 +0530 Subject: [PATCH 8/9] Moving Store outputs to seperate SM --- src/core/cdk/src/initial-setup.ts | 69 ++------------------ src/core/cdk/src/tasks/store-outputs-task.ts | 49 +++++++------- 2 files changed, 32 insertions(+), 86 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 375fb01cf..9a00f0e0a 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -554,7 +554,6 @@ export namespace InitialSetup { return deployTask; }; - const storeOutputsStateMachine = new sfn.StateMachine(this, `${props.acceleratorPrefix}StoreOutputs_sm`, { stateMachineName: `${props.acceleratorPrefix}StoreOutputs_sm`, definition: new StoreOutputsTask(this, 'StoreOutputs', { @@ -564,19 +563,6 @@ export namespace InitialSetup { }); const createStoreOutputTask = (phase: number) => { - - const environment: { [name: string]: string } = { - ACCELERATOR_PHASE: `${phase}`, - 'CONFIG_REPOSITORY_NAME.$': '$.configRepositoryName', - 'CONFIG_FILE_PATH.$': '$.configFilePath', - 'CONFIG_COMMIT_ID.$': '$.configCommitId', - 'ACCELERATOR_BASELINE.$': '$.baseline', - 'CONFIG_ROOT_FILE_PATH.$': '$.configRootFilePath', - ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName, - ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName, - CONFIG_BRANCH_NAME: props.configBranchName, - STACK_OUTPUT_TABLE_NAME: outputsTable.tableName, - }; const storeOutputsTask = new sfn.Task(this, `Store Phase ${phase} Outputs`, { // tslint:disable-next-line: deprecation task: new tasks.StartExecution(storeOutputsStateMachine, { @@ -593,53 +579,10 @@ export namespace InitialSetup { resultPath: 'DISCARD', }); return storeOutputsTask; - - - // const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs ${phase}`, { - // itemsPath: `$.accounts`, - // resultPath: 'DISCARD', - // maxConcurrency: 10, - // parameters: { - // 'account.$': '$$.Map.Item.Value', - // 'regions.$': '$.regions', - // acceleratorPrefix: props.acceleratorPrefix, - // assumeRoleName: props.stateMachineExecutionRole, - // outputsTable: outputsTable.tableName, - // phaseNumber: phase, - // }, - // }); - - // const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs ${phase}`, { - // itemsPath: `$.regions`, - // resultPath: 'DISCARD', - // maxConcurrency: 10, - // parameters: { - // 'account.$': '$.account', - // 'region.$': '$$.Map.Item.Value', - // 'acceleratorPrefix.$': '$.acceleratorPrefix', - // 'assumeRoleName.$': '$.assumeRoleName', - // 'outputsTable.$': '$.outputsTable', - // 'phaseNumber.$': '$.phaseNumber', - // }, - // }); - - // const storeAccountRegionOutputTask = new CodeTask(this, `Store Phase Output ${phase}`, { - // functionProps: { - // code: lambdaCode, - // handler: 'index.storeStackOutputStep', - // role: pipelineRole, - // }, - // resultPath: 'DISCARD', - // }); - - // storeAccountOutputs.iterator(storeAccountRegionOutputs); - // storeAccountRegionOutputs.iterator(storeAccountRegionOutputTask); - - // return storeAccountOutputs; }; // TODO Create separate state machine for deployment - // const deployPhaseRolesTask = createDeploymentTask(-1, false); + const deployPhaseRolesTask = createDeploymentTask(-1, false); const storePreviousOutput = createStoreOutputTask(-1); const deployPhase0Task = createDeploymentTask(0); const storePhase0Output = createStoreOutputTask(0); @@ -838,11 +781,11 @@ export namespace InitialSetup { const commonDefinition = loadOrganizationsTask.startState .next(loadAccountsTask) - // .next(installRolesTask) - // .next(deleteVpcTask) - // .next(loadLimitsTask) - // .next(enableTrustedAccessForServicesTask) - // .next(deployPhaseRolesTask) + .next(installRolesTask) + .next(deleteVpcTask) + .next(loadLimitsTask) + .next(enableTrustedAccessForServicesTask) + .next(deployPhaseRolesTask) .next(storePreviousOutput) .next(deployPhase0Task) .next(storePhase0Output) diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 790a7356f..2eed40e7f 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -31,29 +31,32 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { ); const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs`, { - itemsPath: `$.accounts`, - resultPath: 'DISCARD', - maxConcurrency: 10, - parameters: { - 'account.$': '$$.Map.Item.Value', - 'regions.$': '$.regions', - ...functionPayload, - }, - }); + itemsPath: `$.accounts`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$$.Map.Item.Value', + 'regions.$': '$.regions', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', + }, + }); - const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, { - itemsPath: `$.regions`, - resultPath: 'DISCARD', - maxConcurrency: 10, - parameters: { - 'account.$': '$.account', - 'region.$': '$$.Map.Item.Value', - 'acceleratorPrefix.$': '$.acceleratorPrefix', - 'assumeRoleName.$': '$.assumeRoleName', - 'outputsTable.$': '$.outputsTable', - 'phaseNumber.$': '$.phaseNumber', - }, - }); + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, { + itemsPath: `$.regions`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$.account', + 'region.$': '$$.Map.Item.Value', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', + }, + }); const startTaskResultPath = '$.storeOutputsOutput'; const storeOutputsTask = new CodeTask(scope, `Store Outputs`, { @@ -69,7 +72,7 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { const pass = new sfn.Pass(this, 'Store Outputs Success'); storeAccountOutputs.iterator(storeAccountRegionOutputs); storeAccountRegionOutputs.iterator(storeOutputsTask); - const chain = sfn.Chain.start(storeAccountOutputs).next(pass) + const chain = sfn.Chain.start(storeAccountOutputs).next(pass); this.startState = chain.startState; this.endStates = chain.endStates; From 34846ac41d0f043162c09d90544fe67d63c6ff35 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 1 Sep 2020 16:10:32 +0530 Subject: [PATCH 9/9] Adding encryption to Outputs Table --- src/core/cdk/src/initial-setup.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 9a00f0e0a..f00a697f4 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -95,6 +95,7 @@ export namespace InitialSetup { name: 'id', type: dynamodb.AttributeType.STRING, }, + encryption: dynamodb.TableEncryption.DEFAULT, }); // This is the maximum time before a build times out