aws-samples · nachundu-amzn · Sep 2, 2020 · Aug 31, 2020 · Aug 31, 2020 · Aug 31, 2020
diff --git a/src/core/cdk/package.json b/src/core/cdk/package.json
@@ -40,6 +40,7 @@
     "@aws-cdk/aws-stepfunctions": "1.46.0",
     "@aws-cdk/aws-stepfunctions-tasks": "1.46.0",
     "@aws-cdk/aws-secretsmanager": "1.46.0",
+    "@aws-cdk/aws-dynamodb": "1.46.0",
     "@aws-cdk/core": "1.46.0",
     "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1",
     "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1",

diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts
@@ -4,6 +4,7 @@ import * as iam from '@aws-cdk/aws-iam';
 import * as lambda from '@aws-cdk/aws-lambda';
 import * as s3assets from '@aws-cdk/aws-s3-assets';
 import * as secrets from '@aws-cdk/aws-secretsmanager';
+import * as dynamodb from '@aws-cdk/aws-dynamodb';
 import * as sfn from '@aws-cdk/aws-stepfunctions';
 import * as tasks from '@aws-cdk/aws-stepfunctions-tasks';
 import { CdkDeployProject, PrebuiltCdkDeployProject } from '@aws-accelerator/cdk-accelerator/src/codebuild';
@@ -19,6 +20,7 @@ import { CreateStackTask } from './tasks/create-stack-task';
 import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task';
 import * as fs from 'fs';
 import * as sns from '@aws-cdk/aws-sns';
+import { StoreOutputsTask } from './tasks/store-outputs-task';
 
 export namespace InitialSetup {
   export interface CommonProps {
@@ -84,6 +86,18 @@ export namespace InitialSetup {
       });
       setSecretValue(organizationsSecret, '[]');
 
+      const outputsTable = new dynamodb.Table(this, 'Outputs', {
+        tableName: createName({
+          name: 'Outputs',
+          suffixLength: 0,
+        }),
+        partitionKey: {
+          name: 'id',
+          type: dynamodb.AttributeType.STRING,
+        },
+        encryption: dynamodb.TableEncryption.DEFAULT,
+      });
+
       // This is the maximum time before a build times out
       // The role used by the build should allow this session duration
       const buildTimeout = cdk.Duration.hours(4);
@@ -469,9 +483,7 @@ export namespace InitialSetup {
           'configCommitId.$': '$.configCommitId',
           'organizationalUnits.$': '$.organizationalUnits',
           'accounts.$': '$.accounts',
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
         },
         resultPath: 'DISCARD',
       });
@@ -527,12 +539,8 @@ export namespace InitialSetup {
           ACCELERATOR_PIPELINE_ROLE_NAME: pipelineRole.roleName,
           ACCELERATOR_STATE_MACHINE_NAME: props.stateMachineName,
           CONFIG_BRANCH_NAME: props.configBranchName,
+          STACK_OUTPUT_TABLE_NAME: outputsTable.tableName,
         };
-        if (loadOutputs) {
-          environment['STACK_OUTPUT_BUCKET_NAME.$'] = '$.storeOutput.outputBucketName';
-          environment['STACK_OUTPUT_BUCKET_KEY.$'] = '$.storeOutput.outputBucketKey';
-          environment['STACK_OUTPUT_VERSION.$'] = '$.storeOutput.outputVersion';
-        }
         const deployTask = new sfn.Task(this, `Deploy Phase ${phase}`, {
           // tslint:disable-next-line: deprecation
           task: new tasks.StartExecution(codeBuildStateMachine, {
@@ -547,21 +555,32 @@ export namespace InitialSetup {
         return deployTask;
       };
 
-      const createStoreOutputTask = (phase: number) =>
-        new CodeTask(this, `Store Phase ${phase} Output`, {
-          functionProps: {
-            code: lambdaCode,
-            handler: 'index.storeStackOutputStep',
-            role: pipelineRole,
-          },
-          functionPayload: {
-            acceleratorPrefix: props.acceleratorPrefix,
-            assumeRoleName: props.stateMachineExecutionRole,
-            'accounts.$': '$.accounts',
-            'regions.$': '$.regions',
-          },
-          resultPath: '$.storeOutput',
+      const storeOutputsStateMachine = new sfn.StateMachine(this, `${props.acceleratorPrefix}StoreOutputs_sm`, {
+        stateMachineName: `${props.acceleratorPrefix}StoreOutputs_sm`,
+        definition: new StoreOutputsTask(this, 'StoreOutputs', {
+          lambdaCode,
+          role: pipelineRole,
+        }),
+      });
+
+      const createStoreOutputTask = (phase: number) => {
+        const storeOutputsTask = new sfn.Task(this, `Store Phase ${phase} Outputs`, {
+          // tslint:disable-next-line: deprecation
+          task: new tasks.StartExecution(storeOutputsStateMachine, {
+            integrationPattern: sfn.ServiceIntegrationPattern.SYNC,
+            input: {
+              'accounts.$': '$.accounts',
+              'regions.$': '$.regions',
+              acceleratorPrefix: props.acceleratorPrefix,
+              assumeRoleName: props.stateMachineExecutionRole,
+              outputsTable: outputsTable.tableName,
+              phaseNumber: phase,
+            },
+          }),
+          resultPath: 'DISCARD',
         });
+        return storeOutputsTask;
+      };
 
       // TODO Create separate state machine for deployment
       const deployPhaseRolesTask = createDeploymentTask(-1, false);
@@ -587,9 +606,7 @@ export namespace InitialSetup {
           lambdaPath: 'index.createConfigRecorder',
           name: 'Create Config Recorder',
           functionPayload: {
-            'stackOutputBucketName.$': '$.stackOutputBucketName',
-            'stackOutputBucketKey.$': '$.stackOutputBucketKey',
-            'stackOutputVersion.$': '$.stackOutputVersion',
+            outputTableName: outputsTable.tableName,
           },
         }),
       });
@@ -604,9 +621,7 @@ export namespace InitialSetup {
             'configFilePath.$': '$.configFilePath',
             'configCommitId.$': '$.configCommitId',
             'baseline.$': '$.baseline',
-            'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-            'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-            'stackOutputVersion.$': '$.storeOutput.outputVersion',
+            outputTableName: outputsTable.tableName,
             acceleratorPrefix: props.acceleratorPrefix,
           },
         }),
@@ -626,9 +641,7 @@ export namespace InitialSetup {
           'configRepositoryName.$': '$.configRepositoryName',
           'configFilePath.$': '$.configFilePath',
           'configCommitId.$': '$.configCommitId',
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
         },
         resultPath: 'DISCARD',
       });
@@ -648,9 +661,7 @@ export namespace InitialSetup {
           'configRepositoryName.$': '$.configRepositoryName',
           'configFilePath.$': '$.configFilePath',
           'configCommitId.$': '$.configCommitId',
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
           rdgwScripts,
         },
         resultPath: 'DISCARD',
@@ -668,9 +679,7 @@ export namespace InitialSetup {
           'configRepositoryName.$': '$.configRepositoryName',
           'configFilePath.$': '$.configFilePath',
           'configCommitId.$': '$.configCommitId',
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
         },
         resultPath: 'DISCARD',
       });
@@ -683,9 +692,7 @@ export namespace InitialSetup {
         },
         functionPayload: {
           assumeRoleName: props.stateMachineExecutionRole,
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
         },
         resultPath: 'DISCARD',
       });
@@ -702,9 +709,7 @@ export namespace InitialSetup {
           'configRepositoryName.$': '$.configRepositoryName',
           'configFilePath.$': '$.configFilePath',
           'configCommitId.$': '$.configCommitId',
-          'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-          'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-          'stackOutputVersion.$': '$.storeOutput.outputVersion',
+          outputTableName: outputsTable.tableName,
         },
         resultPath: 'DISCARD',
       });
@@ -728,9 +733,7 @@ export namespace InitialSetup {
             'configRepositoryName.$': '$.configRepositoryName',
             'configFilePath.$': '$.configFilePath',
             'configCommitId.$': '$.configCommitId',
-            'stackOutputBucketName.$': '$.storeOutput.outputBucketName',
-            'stackOutputBucketKey.$': '$.storeOutput.outputBucketKey',
-            'stackOutputVersion.$': '$.storeOutput.outputVersion',
+            outputTableName: outputsTable.tableName,
           },
         }),
         resultPath: 'DISCARD',

diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts
@@ -0,0 +1,80 @@
+import * as cdk from '@aws-cdk/core';
+import * as iam from '@aws-cdk/aws-iam';
+import * as lambda from '@aws-cdk/aws-lambda';
+import * as sfn from '@aws-cdk/aws-stepfunctions';
+import { CodeTask } from '@aws-accelerator/cdk-accelerator/src/stepfunction-tasks';
+
+export namespace StoreOutputsTask {
+  export interface Props {
+    role: iam.IRole;
+    lambdaCode: lambda.Code;
+    functionPayload?: { [key: string]: unknown };
+    waitSeconds?: number;
+  }
+}
+
+export class StoreOutputsTask extends sfn.StateMachineFragment {
+  readonly startState: sfn.State;
+  readonly endStates: sfn.INextable[];
+
+  constructor(scope: cdk.Construct, id: string, props: StoreOutputsTask.Props) {
+    super(scope, id);
+
+    const { role, lambdaCode, functionPayload, waitSeconds = 10 } = props;
+
+    role.addToPrincipalPolicy(
+      new iam.PolicyStatement({
+        effect: iam.Effect.ALLOW,
+        resources: ['*'],
+        actions: ['logs:CreateLogGroup', 'logs:CreateLogStream', 'logs:PutLogEvents'],
+      }),
+    );
+
+    const storeAccountOutputs = new sfn.Map(this, `Store Account Outputs`, {
+      itemsPath: `$.accounts`,
+      resultPath: 'DISCARD',
+      maxConcurrency: 10,
+      parameters: {
+        'account.$': '$$.Map.Item.Value',
+        'regions.$': '$.regions',
+        'acceleratorPrefix.$': '$.acceleratorPrefix',
+        'assumeRoleName.$': '$.assumeRoleName',
+        'outputsTable.$': '$.outputsTable',
+        'phaseNumber.$': '$.phaseNumber',
+      },
+    });
+
+    const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, {
+      itemsPath: `$.regions`,
+      resultPath: 'DISCARD',
+      maxConcurrency: 10,
+      parameters: {
+        'account.$': '$.account',
+        'region.$': '$$.Map.Item.Value',
+        'acceleratorPrefix.$': '$.acceleratorPrefix',
+        'assumeRoleName.$': '$.assumeRoleName',
+        'outputsTable.$': '$.outputsTable',
+        'phaseNumber.$': '$.phaseNumber',
+      },
+    });
+
+    const startTaskResultPath = '$.storeOutputsOutput';
+    const storeOutputsTask = new CodeTask(scope, `Store Outputs`, {
+      resultPath: startTaskResultPath,
+      functionPayload,
+      functionProps: {
+        role,
+        code: lambdaCode,
+        handler: 'index.storeStackOutputStep',
+      },
+    });
+
+    const pass = new sfn.Pass(this, 'Store Outputs Success');
+    storeAccountOutputs.iterator(storeAccountRegionOutputs);
+    storeAccountRegionOutputs.iterator(storeOutputsTask);
+    const chain = sfn.Chain.start(storeAccountOutputs).next(pass);
+
+    this.startState = chain.startState;
+    this.endStates = chain.endStates;
+  }
+}
diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts
@@ -1,6 +1,7 @@
 import * as aws from 'aws-sdk';
 import { Account } from '@aws-accelerator/common-outputs/src/accounts';
 import { STS } from '@aws-accelerator/common/src/aws/sts';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
 import {
   StackOutput,
   getStackOutput,
@@ -11,38 +12,21 @@ import { CloudTrail } from '@aws-accelerator/common/src/aws/cloud-trail';
 import { PutEventSelectorsRequest, UpdateTrailRequest } from 'aws-sdk/clients/cloudtrail';
 import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
 import { LoadConfigurationInput } from './load-configuration-step';
-import { S3 } from '@aws-accelerator/common/src/aws/s3';
+import { loadOutputs } from './utils/load-outputs';
 
 interface AccountDefaultSettingsInput extends LoadConfigurationInput {
   assumeRoleName: string;
   accounts: Account[];
-  stackOutputBucketName: string;
-  stackOutputBucketKey: string;
-  stackOutputVersion: string;
+  outputTableName: string;
 }
 
-const s3 = new S3();
+const dynamodb = new DynamoDB();
 
 export const handler = async (input: AccountDefaultSettingsInput) => {
   console.log('Setting account level defaults for all accounts in an organization ...');
   console.log(JSON.stringify(input, null, 2));
 
-  const {
-    assumeRoleName,
-    accounts,
-    configRepositoryName,
-    configFilePath,
-    configCommitId,
-    stackOutputBucketName,
-    stackOutputBucketKey,
-    stackOutputVersion,
-  } = input;
-
-  const outputsString = await s3.getObjectBodyAsString({
-    Bucket: stackOutputBucketName,
-    Key: stackOutputBucketKey,
-    VersionId: stackOutputVersion,
-  });
+  const { assumeRoleName, accounts, configRepositoryName, configFilePath, configCommitId, outputTableName } = input;
 
   // Retrieve Configuration from Code Commit with specific commitId
   const acceleratorConfig = await loadAcceleratorConfig({
@@ -53,7 +37,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => {
 
   const logAccountKey = acceleratorConfig.getMandatoryAccountKey('central-log');
 
-  const outputs = JSON.parse(outputsString) as StackOutput[];
+  const outputs = await loadOutputs(outputTableName, dynamodb);
 
   const sts = new STS();
 

diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts
@@ -2,21 +2,19 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts';
 import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations';
 import { LoadConfigurationInput } from './load-configuration-step';
 import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load';
-import { S3 } from '@aws-accelerator/common/src/aws/s3';
-import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output';
+import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb';
 import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts';
 import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp';
+import { loadOutputs } from './utils/load-outputs';
 
 interface AddScpInput extends LoadConfigurationInput {
   acceleratorPrefix: string;
   accounts: Account[];
   organizationalUnits: OrganizationalUnit[];
-  stackOutputBucketName: string;
-  stackOutputBucketKey: string;
-  stackOutputVersion: string;
+  outputTableName: string;
 }
 
-const s3 = new S3();
+const dynamodb = new DynamoDB();
 
 export const handler = async (input: AddScpInput) => {
   console.log(`Adding service control policy to organization...`);
@@ -29,9 +27,7 @@ export const handler = async (input: AddScpInput) => {
     configRepositoryName,
     configFilePath,
     configCommitId,
-    stackOutputBucketName,
-    stackOutputBucketKey,
-    stackOutputVersion,
+    outputTableName,
   } = input;
 
   // Retrieve Configuration from Code Commit with specific commitId
@@ -43,12 +39,7 @@ export const handler = async (input: AddScpInput) => {
   const organizationAdminRole = config['global-options']['organization-admin-role']!;
   const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole);
 
-  const outputsString = await s3.getObjectBodyAsString({
-    Bucket: stackOutputBucketName,
-    Key: stackOutputBucketKey,
-    VersionId: stackOutputVersion,
-  });
-  const outputs = JSON.parse(outputsString) as StackOutput[];
+  const outputs = await loadOutputs(outputTableName, dynamodb);
 
   // Find the SCP artifact output
   const artifactOutput = ArtifactOutputFinder.findOneByName({