diff --git a/src/core/cdk/package.json b/src/core/cdk/package.json index 4a35cb3c4..0c526cea1 100644 --- a/src/core/cdk/package.json +++ b/src/core/cdk/package.json @@ -40,6 +40,7 @@ "@aws-cdk/aws-stepfunctions": "1.46.0", "@aws-cdk/aws-stepfunctions-tasks": "1.46.0", "@aws-cdk/aws-secretsmanager": "1.46.0", + "@aws-cdk/aws-dynamodb": "1.46.0", "@aws-cdk/core": "1.46.0", "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 04ed554c6..f00a697f4 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -4,6 +4,7 @@ import * as iam from '@aws-cdk/aws-iam'; import * as lambda from '@aws-cdk/aws-lambda'; import * as s3assets from '@aws-cdk/aws-s3-assets'; import * as secrets from '@aws-cdk/aws-secretsmanager'; +import * as dynamodb from '@aws-cdk/aws-dynamodb'; import * as sfn from '@aws-cdk/aws-stepfunctions'; import * as tasks from '@aws-cdk/aws-stepfunctions-tasks'; import { CdkDeployProject, PrebuiltCdkDeployProject } from '@aws-accelerator/cdk-accelerator/src/codebuild'; @@ -19,6 +20,7 @@ import { CreateStackTask } from './tasks/create-stack-task'; import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task'; import * as fs from 'fs'; import * as sns from '@aws-cdk/aws-sns'; +import { StoreOutputsTask } from './tasks/store-outputs-task'; export namespace InitialSetup { export interface CommonProps { @@ -84,6 +86,18 @@ export namespace InitialSetup { }); setSecretValue(organizationsSecret, '[]'); + const outputsTable = new dynamodb.Table(this, 'Outputs', { + tableName: createName({ + name: 'Outputs', + suffixLength: 0, + }), + partitionKey: { + name: 'id', + type: dynamodb.AttributeType.STRING, + }, + encryption: dynamodb.TableEncryption.DEFAULT, + }); + // This is the maximum time before a build times out // The role used by the build should allow this session duration const buildTimeout = cdk.Duration.hours(4); @@ -469,9 +483,7 @@ export namespace InitialSetup { 'configCommitId.$': '$.configCommitId', 'organizationalUnits.$': '$.organizationalUnits', 'accounts.$': '$.accounts', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -527,12 +539,8 @@ export namespace InitialSetup { ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName, ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName, CONFIG_BRANCH_NAME: props.configBranchName, + STACK_OUTPUT_TABLE_NAME: outputsTable.tableName, }; - if (loadOutputs) { - environment['STACK_OUTPUT_BUCKET_NAME.$'] = '$.storeOutput.outputBucketName'; - environment['STACK_OUTPUT_BUCKET_KEY.$'] = '$.storeOutput.outputBucketKey'; - environment['STACK_OUTPUT_VERSION.$'] = '$.storeOutput.outputVersion'; - } const deployTask = new sfn.Task(this, `Deploy Phase ${phase}`, { // tslint:disable-next-line: deprecation task: new tasks.StartExecution(codeBuildStateMachine, { @@ -547,21 +555,32 @@ export namespace InitialSetup { return deployTask; }; - const createStoreOutputTask = (phase: number) => - new CodeTask(this, `Store Phase ${phase} Output`, { - functionProps: { - code: lambdaCode, - handler: 'index.storeStackOutputStep', - role: pipelineRole, - }, - functionPayload: { - acceleratorPrefix: props.acceleratorPrefix, - assumeRoleName: props.stateMachineExecutionRole, - 'accounts.$': '$.accounts', - 'regions.$': '$.regions', - }, - resultPath: '$.storeOutput', + const storeOutputsStateMachine = new sfn.StateMachine(this, `${props.acceleratorPrefix}StoreOutputs_sm`, { + stateMachineName: `${props.acceleratorPrefix}StoreOutputs_sm`, + definition: new StoreOutputsTask(this, 'StoreOutputs', { + lambdaCode, + role: pipelineRole, + }), + }); + + const createStoreOutputTask = (phase: number) => { + const storeOutputsTask = new sfn.Task(this, `Store Phase ${phase} Outputs`, { + // tslint:disable-next-line: deprecation + task: new tasks.StartExecution(storeOutputsStateMachine, { + integrationPattern: sfn.ServiceIntegrationPattern.SYNC, + input: { + 'accounts.$': '$.accounts', + 'regions.$': '$.regions', + acceleratorPrefix: props.acceleratorPrefix, + assumeRoleName: props.stateMachineExecutionRole, + outputsTable: outputsTable.tableName, + phaseNumber: phase, + }, + }), + resultPath: 'DISCARD', }); + return storeOutputsTask; + }; // TODO Create separate state machine for deployment const deployPhaseRolesTask = createDeploymentTask(-1, false); @@ -587,9 +606,7 @@ export namespace InitialSetup { lambdaPath: 'index.createConfigRecorder', name: 'Create Config Recorder', functionPayload: { - 'stackOutputBucketName.$': '$.stackOutputBucketName', - 'stackOutputBucketKey.$': '$.stackOutputBucketKey', - 'stackOutputVersion.$': '$.stackOutputVersion', + outputTableName: outputsTable.tableName, }, }), }); @@ -604,9 +621,7 @@ export namespace InitialSetup { 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', 'baseline.$': '$.baseline', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, acceleratorPrefix: props.acceleratorPrefix, }, }), @@ -626,9 +641,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -648,9 +661,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, rdgwScripts, }, resultPath: 'DISCARD', @@ -668,9 +679,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -683,9 +692,7 @@ export namespace InitialSetup { }, functionPayload: { assumeRoleName: props.stateMachineExecutionRole, - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -702,9 +709,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', }); @@ -728,9 +733,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'stackOutputBucketName.$': '$.storeOutput.outputBucketName', - 'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey', - 'stackOutputVersion.$': '$.storeOutput.outputVersion', + outputTableName: outputsTable.tableName, }, }), resultPath: 'DISCARD', diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts new file mode 100644 index 000000000..2eed40e7f --- /dev/null +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -0,0 +1,80 @@ +import * as cdk from '@aws-cdk/core'; +import * as iam from '@aws-cdk/aws-iam'; +import * as lambda from '@aws-cdk/aws-lambda'; +import * as sfn from '@aws-cdk/aws-stepfunctions'; +import { CodeTask } from '@aws-accelerator/cdk-accelerator/src/stepfunction-tasks'; + +export namespace StoreOutputsTask { + export interface Props { + role: iam.IRole; + lambdaCode: lambda.Code; + functionPayload?: { [key: string]: unknown }; + waitSeconds?: number; + } +} + +export class StoreOutputsTask extends sfn.StateMachineFragment { + readonly startState: sfn.State; + readonly endStates: sfn.INextable[]; + + constructor(scope: cdk.Construct, id: string, props: StoreOutputsTask.Props) { + super(scope, id); + + const { role, lambdaCode, functionPayload, waitSeconds = 10 } = props; + + role.addToPrincipalPolicy( + new iam.PolicyStatement({ + effect: iam.Effect.ALLOW, + resources: ['*'], + actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'], + }), + ); + + const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs`, { + itemsPath: `$.accounts`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$$.Map.Item.Value', + 'regions.$': '$.regions', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', + }, + }); + + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, { + itemsPath: `$.regions`, + resultPath: 'DISCARD', + maxConcurrency: 10, + parameters: { + 'account.$': '$.account', + 'region.$': '$$.Map.Item.Value', + 'acceleratorPrefix.$': '$.acceleratorPrefix', + 'assumeRoleName.$': '$.assumeRoleName', + 'outputsTable.$': '$.outputsTable', + 'phaseNumber.$': '$.phaseNumber', + }, + }); + + const startTaskResultPath = '$.storeOutputsOutput'; + const storeOutputsTask = new CodeTask(scope, `Store Outputs`, { + resultPath: startTaskResultPath, + functionPayload, + functionProps: { + role, + code: lambdaCode, + handler: 'index.storeStackOutputStep', + }, + }); + + const pass = new sfn.Pass(this, 'Store Outputs Success'); + storeAccountOutputs.iterator(storeAccountRegionOutputs); + storeAccountRegionOutputs.iterator(storeOutputsTask); + const chain = sfn.Chain.start(storeAccountOutputs).next(pass); + + this.startState = chain.startState; + this.endStates = chain.endStates; + } +} diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts index 6e8dc07ff..1ef165997 100644 --- a/src/core/runtime/src/account-default-settings-step.ts +++ b/src/core/runtime/src/account-default-settings-step.ts @@ -1,6 +1,7 @@ import * as aws from 'aws-sdk'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { StackOutput, getStackOutput, @@ -11,38 +12,21 @@ import { CloudTrail } from '@aws-accelerator/common/src/aws/cloud-trail'; import { PutEventSelectorsRequest, UpdateTrailRequest } from 'aws-sdk/clients/cloudtrail'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { loadOutputs } from './utils/load-outputs'; interface AccountDefaultSettingsInput extends LoadConfigurationInput { assumeRoleName: string; accounts: Account[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: AccountDefaultSettingsInput) => { console.log('Setting account level defaults for all accounts in an organization ...'); console.log(JSON.stringify(input, null, 2)); - const { - assumeRoleName, - accounts, - configRepositoryName, - configFilePath, - configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, - } = input; - - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); + const { assumeRoleName, accounts, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ @@ -53,7 +37,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => { const logAccountKey = acceleratorConfig.getMandatoryAccountKey('central-log'); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index 906420aec..ba018ad57 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -2,21 +2,19 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts'; import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; +import { loadOutputs } from './utils/load-outputs'; interface AddScpInput extends LoadConfigurationInput { acceleratorPrefix: string; accounts: Account[]; organizationalUnits: OrganizationalUnit[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: AddScpInput) => { console.log(`Adding service control policy to organization...`); @@ -29,9 +27,7 @@ export const handler = async (input: AddScpInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -43,12 +39,7 @@ export const handler = async (input: AddScpInput) => { const organizationAdminRole = config['global-options']['organization-admin-role']!; const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // Find the SCP artifact output const artifactOutput = ArtifactOutputFinder.findOneByName({ diff --git a/src/core/runtime/src/add-tags-to-shared-resources-step.ts b/src/core/runtime/src/add-tags-to-shared-resources-step.ts index a2c195ca7..5473b8889 100644 --- a/src/core/runtime/src/add-tags-to-shared-resources-step.ts +++ b/src/core/runtime/src/add-tags-to-shared-resources-step.ts @@ -1,15 +1,14 @@ import { TagResources } from '@aws-accelerator/common/src/aws/resource-tagging'; import { STS } from '@aws-accelerator/common/src/aws/sts'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { loadOutputs } from './utils/load-outputs'; const ALLOWED_RESOURCE_TYPES = ['subnet', 'security-group', 'vpc', 'tgw-attachment']; interface CreateTagsRequestInput { assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface Tag { @@ -26,22 +25,16 @@ interface AddTagToResourceOutput { type AddTagToResourceOutputs = AddTagToResourceOutput[]; -const s3 = new S3(); +const dynamodb = new DynamoDB(); const sts = new STS(); export const handler = async (input: CreateTagsRequestInput) => { console.log(`Adding tags to shared resource...`); console.log(JSON.stringify(input, null, 2)); - const { assumeRoleName, stackOutputBucketName, stackOutputBucketKey, stackOutputVersion } = input; - - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const { assumeRoleName, outputTableName } = input; + const outputs = await loadOutputs(outputTableName, dynamodb); const addTagsToResourcesOutputs: AddTagToResourceOutputs[] = getStackJsonOutput(outputs, { outputType: 'AddTagsToResources', }); diff --git a/src/core/runtime/src/associate-hosted-zones-step.ts b/src/core/runtime/src/associate-hosted-zones-step.ts index 9a06b89b6..8f4fa104f 100644 --- a/src/core/runtime/src/associate-hosted-zones-step.ts +++ b/src/core/runtime/src/associate-hosted-zones-step.ts @@ -1,5 +1,5 @@ import * as r53 from 'aws-sdk/clients/route53'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { getStackJsonOutput, StackOutput, ResolversOutput } from '@aws-accelerator/common-outputs/src/stack-output'; @@ -9,13 +9,12 @@ import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { throttlingBackOff } from '@aws-accelerator/common/src/aws/backoff'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; +import { loadOutputs } from './utils/load-outputs'; interface AssociateHostedZonesInput extends LoadConfigurationInput { accounts: Account[]; assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } type ResolversOutputs = ResolversOutput[]; @@ -42,23 +41,14 @@ interface AccountRule { // Hosted zone ID is in the form of `/hostedzone/Z0181099DGX53XMU1D7S` const hostedZoneIdRegex = /\/hostedzone\/([\d\w]+)/; -const s3 = new S3(); +const dynamodb = new DynamoDB(); const sts = new STS(); export const handler = async (input: AssociateHostedZonesInput) => { console.log(`Associating Hosted Zones with VPC...`); console.log(JSON.stringify(input, null, 2)); - const { - configRepositoryName, - accounts, - assumeRoleName, - configCommitId, - configFilePath, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, - } = input; + const { configRepositoryName, accounts, assumeRoleName, configCommitId, configFilePath, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -67,12 +57,7 @@ export const handler = async (input: AssociateHostedZonesInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // get the private zones from global-options const globalOptionsConfig = config['global-options']; diff --git a/src/core/runtime/src/create-adconnector/create.ts b/src/core/runtime/src/create-adconnector/create.ts index d1d8b8632..710ecb288 100644 --- a/src/core/runtime/src/create-adconnector/create.ts +++ b/src/core/runtime/src/create-adconnector/create.ts @@ -1,6 +1,6 @@ import { DirectoryService } from '@aws-accelerator/common/src/aws/directory-service'; import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { createMadUserPasswordSecretName, MadOutput } from '@aws-accelerator/common-outputs/src/mad'; @@ -8,6 +8,7 @@ import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; +import { loadOutputs } from '../utils/load-outputs'; const VALID_STATUSES: string[] = ['Requested', 'Creating', 'Created', 'Active', 'Inoperable', 'Impaired', 'Restoring']; @@ -18,9 +19,7 @@ interface AdConnectorInput extends LoadConfigurationInput { configRepositoryName: string; configFilePath: string; configCommitId: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } export interface AdConnectorOutput { @@ -29,7 +28,7 @@ export interface AdConnectorOutput { assumeRoleName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); const secrets = new SecretsManager(); const sts = new STS(); @@ -44,9 +43,7 @@ export const handler = async (input: AdConnectorInput) => { configRepositoryName, configFilePath, configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -56,12 +53,7 @@ export const handler = async (input: AdConnectorInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); const adConnectorOutputs: AdConnectorOutput[] = []; for (const [accountKey, mandatoryConfig] of acceleratorConfig.getMandatoryAccountConfigs()) { diff --git a/src/core/runtime/src/create-config-recorder/create.ts b/src/core/runtime/src/create-config-recorder/create.ts index f20e3a8d3..761ed709c 100644 --- a/src/core/runtime/src/create-config-recorder/create.ts +++ b/src/core/runtime/src/create-config-recorder/create.ts @@ -1,21 +1,20 @@ import { ConfigService } from '@aws-accelerator/common/src/aws/configservice'; import { ConfigurationRecorder } from 'aws-sdk/clients/configservice'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; -import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { LoadConfigurationInput } from '../load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { createConfigRecorderName, createAggregatorName } from '@aws-accelerator/common-outputs/src/config'; import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-role'; +import { loadOutputs } from '../utils/load-outputs'; interface ConfigServiceInput extends LoadConfigurationInput { account: Account; assumeRoleName: string; acceleratorPrefix: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface LogBucketOutputType { @@ -36,7 +35,7 @@ const CustomErrorMessage = [ ]; const sts = new STS(); -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: ConfigServiceInput): Promise => { console.log(`Enable Config Recorder in account ...`); @@ -48,17 +47,10 @@ export const handler = async (input: ConfigServiceInput): Promise => { configFilePath, configCommitId, acceleratorPrefix, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, + outputTableName, } = input; - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/enable-directory-sharing-step.ts b/src/core/runtime/src/enable-directory-sharing-step.ts index b7c19da9c..b7d9c030b 100644 --- a/src/core/runtime/src/enable-directory-sharing-step.ts +++ b/src/core/runtime/src/enable-directory-sharing-step.ts @@ -1,36 +1,26 @@ import { DirectoryService } from '@aws-accelerator/common/src/aws/directory-service'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { MadOutput } from '@aws-accelerator/common-outputs/src/mad'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; +import { loadOutputs } from './utils/load-outputs'; interface ShareDirectoryInput extends LoadConfigurationInput { accounts: Account[]; assumeRoleName: string; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } -const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: ShareDirectoryInput) => { console.log(`Sharing MAD to another account ...`); console.log(JSON.stringify(input, null, 2)); - const { - accounts, - assumeRoleName, - configRepositoryName, - configFilePath, - configCommitId, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, - } = input; + const { accounts, assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ @@ -39,12 +29,7 @@ export const handler = async (input: ShareDirectoryInput) => { commitId: configCommitId, }); - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const outputs = await loadOutputs(outputTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index 71636db54..1abf7e454 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -1,144 +1,83 @@ -import * as aws from 'aws-sdk'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { collectAsync } from '@aws-accelerator/common/src/util/generator'; -import { CentralBucketOutputFinder } from '@aws-accelerator/common-outputs/src/central-bucket'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; export interface StoreStackOutputInput { acceleratorPrefix: string; assumeRoleName: string; - accounts: Account[]; - regions: string[]; + account: Account; + region: string; + outputsTable: string; + phaseNumber: number; } -const s3 = new S3(); const sts = new STS(); +const dynamodb = new DynamoDB(); export const handler = async (input: StoreStackOutputInput) => { console.log(`Storing stack output...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, assumeRoleName, accounts, regions } = input; - - const outputsAsyncIterable = getOutputsForAccountsAndRegions({ - acceleratorPrefix, - accounts, - assumeRoleName, - regions, - }); - const outputs = await collectAsync(outputsAsyncIterable); - - // Find the central output bucket in outputs - const centralBucketOutput = CentralBucketOutputFinder.tryFindOne({ - outputs, + const { acceleratorPrefix, assumeRoleName, account, region, outputsTable, phaseNumber } = input; + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const cfn = new CloudFormation(credentials, region); + const stacks = cfn.listStacksGenerator({ + StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], }); + const outputs: StackOutput[] = []; + for await (const summary of stacks) { + if (!summary.StackName.match(`${acceleratorPrefix}(.*)-Phase${phaseNumber}`)) { + console.warn(`Skipping stack with name "${summary.StackName}"`); + continue; + } + const stack = await cfn.describeStack(summary.StackName); + if (!stack) { + console.warn(`Could not load stack with name "${summary.StackName}"`); + continue; + } + const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); + if (!acceleratorTag) { + console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}"`); + continue; + } - if (!centralBucketOutput) { - console.log(`Didn't find "centralBucket" in existing outputs, might be first run of Accelerator`); + console.debug(`Storing outputs for stack with name "${summary.StackName}"`); + stack.Outputs?.forEach(output => + outputs.push({ + accountKey: account.key, + outputKey: `${output.OutputKey}sjkdh`, + outputValue: output.OutputValue, + outputDescription: output.Description, + outputExportName: output.ExportName, + region, + }), + ); + } + if (outputs.length === 0) { + console.warn(`No outputs found for Account: ${account.key} and Region: ${region}`); + await dynamodb.deleteItem({ + TableName: outputsTable, + Key: { + id: { S: `${account.key}-${region}-${phaseNumber}` }, + }, + }); return { status: 'SUCCESS', - outputBucketName: '', - outputBucketKey: '', - outputVersion: '', }; } - - console.log(`Writing outputs to s3://${centralBucketOutput.bucketName}/outputs.json`); - - // Store outputs on S3 - const response = await s3.putObject({ - Bucket: centralBucketOutput.bucketName, - Key: 'outputs.json', - Body: JSON.stringify(outputs), + await dynamodb.putItem({ + Item: { + id: { S: `${account.key}-${region}-${phaseNumber}` }, + accountKey: { S: account.key }, + region: { S: region }, + phase: { N: `${phaseNumber}` }, + outputValue: { S: JSON.stringify(outputs) }, + }, + TableName: outputsTable, }); - return { status: 'SUCCESS', - outputBucketName: centralBucketOutput.bucketName, - outputBucketKey: 'outputs.json', - outputVersion: response.VersionId, }; }; - -/** - * Find all outputs in the given accounts and regions. - */ -async function* getOutputsForAccountsAndRegions(props: { - acceleratorPrefix: string; - accounts: Account[]; - assumeRoleName: string; - regions: string[]; -}): AsyncIterableIterator { - const { acceleratorPrefix, accounts, assumeRoleName, regions } = props; - const outputsListPromises = []; - for (const account of accounts) { - const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); - for (const region of regions) { - const outputsListPromise = getOutputsForRegion({ - acceleratorPrefix, - accountKey: account.key, - credentials, - region, - }); - outputsListPromises.push(outputsListPromise); - } - } - for (const outputsList of outputsListPromises) { - yield* outputsList; - } -} - -/** - * Find all outputs in the given account and region. - */ -async function* getOutputsForRegion(props: { - acceleratorPrefix: string; - accountKey: string; - credentials: aws.Credentials; - region: string; -}): AsyncIterableIterator { - const { acceleratorPrefix, accountKey, credentials, region } = props; - - try { - const cfn = new CloudFormation(credentials, region); - - const stacks = cfn.listStacksGenerator({ - StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], - }); - - for await (const summary of stacks) { - if (!summary.StackName.startsWith(acceleratorPrefix)) { - console.warn(`Skipping stack with name "${summary.StackName}"`); - continue; - } - const stack = await cfn.describeStack(summary.StackName); - if (!stack) { - console.warn(`Could not load stack with name "${summary.StackName}"`); - continue; - } - const acceleratorTag = stack.Tags?.find(t => t.Key === 'Accelerator'); - if (!acceleratorTag) { - console.warn(`Could not find Accelerator tag in stack with name "${summary.StackName}" in region ${region}`); - continue; - } - - console.debug(`Storing outputs for stack with name "${summary.StackName}" in region ${region}`); - for (const output of stack.Outputs || []) { - yield { - accountKey, - region, - outputKey: output.OutputKey, - outputValue: output.OutputValue, - outputDescription: output.Description, - outputExportName: output.ExportName, - }; - } - } - } catch (e) { - console.warn(`Cannot find outputs in account "${accountKey}" and region "${region}": ${e}`); - } -} diff --git a/src/core/runtime/src/utils/load-outputs.ts b/src/core/runtime/src/utils/load-outputs.ts new file mode 100644 index 000000000..83850d0ae --- /dev/null +++ b/src/core/runtime/src/utils/load-outputs.ts @@ -0,0 +1,18 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; + +export async function loadOutputs(tableName: string, client: DynamoDB): Promise { + const outputs: StackOutput[] = []; + const outputsResponse = await client.scan({ + TableName: tableName, + }); + if (!outputsResponse.Items) { + console.warn(`Did not find outputs in DynamoDB table "${tableName}"`); + return []; + } + for (const item of outputsResponse.Items) { + const cVal = JSON.parse(item.outputValue.S!); + outputs.push(...cVal); + } + return outputs; +} diff --git a/src/core/runtime/src/verify-files-step.ts b/src/core/runtime/src/verify-files-step.ts index 4f02bfc03..c500d2ffa 100644 --- a/src/core/runtime/src/verify-files-step.ts +++ b/src/core/runtime/src/verify-files-step.ts @@ -1,16 +1,16 @@ import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts'; import { CentralBucketOutputFinder } from '@aws-accelerator/common-outputs/src/central-bucket'; import * as c from '@aws-accelerator/common-config/src'; +import { loadOutputs } from './utils/load-outputs'; interface VerifyFilesInput extends LoadConfigurationInput { rdgwScripts: string[]; - stackOutputBucketName: string; - stackOutputBucketKey: string; - stackOutputVersion: string; + outputTableName: string; } interface RdgwArtifactsOutput { @@ -21,27 +21,15 @@ interface RdgwArtifactsOutput { } const s3 = new S3(); +const dynamodb = new DynamoDB(); export const handler = async (input: VerifyFilesInput) => { console.log('Validate existence of all required files ...'); console.log(JSON.stringify(input, null, 2)); - const { - configRepositoryName, - configFilePath, - configCommitId, - rdgwScripts, - stackOutputBucketName, - stackOutputBucketKey, - stackOutputVersion, - } = input; - - const outputsString = await s3.getObjectBodyAsString({ - Bucket: stackOutputBucketName, - Key: stackOutputBucketKey, - VersionId: stackOutputVersion, - }); - const outputs = JSON.parse(outputsString) as StackOutput[]; + const { configRepositoryName, configFilePath, configCommitId, rdgwScripts, outputTableName } = input; + + const outputs = await loadOutputs(outputTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/deployments/cdk/src/utils/outputs.ts b/src/deployments/cdk/src/utils/outputs.ts index c0e81dae0..ee4739948 100644 --- a/src/deployments/cdk/src/utils/outputs.ts +++ b/src/deployments/cdk/src/utils/outputs.ts @@ -1,9 +1,9 @@ import * as fs from 'fs'; import * as path from 'path'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; -import { S3 } from '@aws-accelerator/common/src/aws/s3'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -const s3 = new S3(); +const dynamodb = new DynamoDB(); export async function loadStackOutputs(): Promise { if (process.env.CONFIG_MODE === 'development') { @@ -15,29 +15,23 @@ export async function loadStackOutputs(): Promise { return JSON.parse(contents.toString()); } - const outputBucketName = process.env.STACK_OUTPUT_BUCKET_NAME; - const outputBucketKey = process.env.STACK_OUTPUT_BUCKET_KEY; - const outputVersion = process.env.STACK_OUTPUT_VERSION; - if (!outputBucketName || !outputBucketKey || !outputVersion) { - console.warn( - `The environment variable "STACK_OUTPUT_BUCKET_NAME", "STACK_OUTPUT_BUCKET_KEY", "STACK_OUTPUT_VERSION" need to be set`, - ); + const outputTableName = process.env.STACK_OUTPUT_TABLE_NAME; + if (!outputTableName) { + console.warn(`The environment variable "STACK_OUTPUT_TABLE_NAME" need to be set`); return []; } - const outputsJson = await s3.getObjectBodyAsString({ - Bucket: outputBucketName, - Key: outputBucketKey, - VersionId: outputVersion, + const outputs: StackOutput[] = []; + const outputsResponse = await dynamodb.scan({ + TableName: outputTableName, }); - if (!outputsJson) { - console.warn(`Cannot find outputs "s3://${outputBucketName}${outputBucketKey}"`); + if (!outputsResponse.Items) { + console.warn(`Did not find outputs in DynamoDB table "${outputTableName}"`); return []; } - try { - return JSON.parse(outputsJson); - } catch (e) { - console.warn(`Cannot parse outputs "s3://${outputBucketName}${outputBucketKey}"`); - return []; + for (const item of outputsResponse.Items) { + const cVal = JSON.parse(item.outputValue.S!); + outputs.push(...cVal); } + return outputs; } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts new file mode 100644 index 000000000..87325c0fe --- /dev/null +++ b/src/lib/common/src/aws/dynamodb.ts @@ -0,0 +1,33 @@ +import aws from './aws-client'; +import * as dynamodb from 'aws-sdk/clients/dynamodb'; +import { throttlingBackOff } from './backoff'; + +export class DynamoDB { + private readonly client: aws.DynamoDB; + + constructor(credentials?: aws.Credentials) { + this.client = new aws.DynamoDB({ + credentials, + }); + } + + async createTable(props: dynamodb.CreateTableInput): Promise { + await throttlingBackOff(() => this.client.createTable(props).promise()); + } + + async putItem(props: dynamodb.PutItemInput): Promise { + await throttlingBackOff(() => this.client.putItem(props).promise()); + } + + async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { + await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + } + + async scan(props: dynamodb.ScanInput): Promise { + return throttlingBackOff(() => this.client.scan(props).promise()); + } + + async deleteItem(props: dynamodb.DeleteItemInput): Promise { + await throttlingBackOff(() => this.client.deleteItem(props).promise()); + } +}