From a4a9310c7b13c0fac94bbd9f8c521df2180abede Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 16:29:00 +0530 Subject: [PATCH 01/20] Initial Push --- src/core/cdk/src/initial-setup.ts | 31 ++++---- .../src/account-default-settings-step.ts | 7 +- .../runtime/src/add-role-to-kms-key-step.ts | 12 ++- src/core/runtime/src/add-scp-step.ts | 14 +++- .../src/associate-hosted-zones-step.ts | 7 +- .../configuration/load-landing-zone-config.ts | 2 +- .../load-organizations-config.ts | 4 +- .../runtime/src/create-adconnector/create.ts | 6 +- src/core/runtime/src/detach-quarantine-scp.ts | 9 ++- .../src/enable-directory-sharing-step.ts | 12 +-- ...enable-trusted-access-for-services-step.ts | 11 ++- src/core/runtime/src/load-accounts-step.ts | 78 +++++++++++-------- src/core/runtime/src/load-limits-step.ts | 11 +-- .../src/notify-statemachine-success.ts | 8 +- .../runtime/src/store-stack-output-step.ts | 4 +- src/core/runtime/src/utils/load-accounts.ts | 24 ++++++ src/lib/common-outputs/src/accounts.ts | 7 +- 17 files changed, 164 insertions(+), 83 deletions(-) create mode 100644 src/core/runtime/src/utils/load-accounts.ts diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 42a8bf153..3fc464cba 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -313,7 +313,7 @@ export namespace InitialSetup { 'configFilePath.$': '$.configuration.configFilePath', 'configCommitId.$': '$.configuration.configCommitId', }, - resultPath: '$.configuration.organizationalUnits', + resultPath: 'DISCARD', }); const loadAccountsTask = new CodeTask(this, 'Load Accounts', { @@ -326,7 +326,14 @@ export namespace InitialSetup { parametersTableName: parametersTable.tableName, itemId: 'accounts', accountsItemsCountId: 'accounts-items-count', - 'configuration.$': '$.configuration', + // Sending required Inputs seperately to omit unnecesary inputs from SM Input + 'configRepositoryName.$': '$.configuration.configRepositoryName', + 'configFilePath.$': '$.configuration.configFilePath', + 'configCommitId.$': '$.configuration.configCommitId', + 'acceleratorVersion.$': '$.configuration.acceleratorVersion', + 'baseline.$': '$.configuration.baseline', + 'regions.$': '$.configuration.regions', + 'accounts.$': '$.configuration.accounts', }, resultPath: '$', }); @@ -443,7 +450,6 @@ export namespace InitialSetup { parametersTableName: parametersTable.tableName, itemId: 'limits', assumeRoleName: props.stateMachineExecutionRole, - 'accounts.$': '$.accounts', }, resultPath: '$.limits', }); @@ -479,8 +485,7 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - 'organizationalUnits.$': '$.organizationalUnits', - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, outputTableName: outputsTable.tableName, }, resultPath: 'DISCARD', @@ -496,7 +501,7 @@ export namespace InitialSetup { }, functionPayload: { acceleratorPrefix: props.acceleratorPrefix, - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, }, resultPath: 'DISCARD', }); @@ -509,7 +514,7 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', @@ -635,7 +640,7 @@ export namespace InitialSetup { }, functionPayload: { assumeRoleName: props.stateMachineExecutionRole, - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', @@ -655,7 +660,7 @@ export namespace InitialSetup { }, functionPayload: { assumeRoleName: props.stateMachineExecutionRole, - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', @@ -672,7 +677,7 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, assumeRoleName: props.stateMachineExecutionRole, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', @@ -702,7 +707,7 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, assumeRoleName: props.stateMachineExecutionRole, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', @@ -726,7 +731,7 @@ export namespace InitialSetup { integrationPattern: sfn.ServiceIntegrationPattern.SYNC, input: { acceleratorPrefix: props.acceleratorPrefix, - 'accounts.$': '$.accounts', + parametersTableName: parametersTable.tableName, assumeRoleName: props.stateMachineExecutionRole, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', @@ -860,7 +865,7 @@ export namespace InitialSetup { }, functionPayload: { notificationTopicArn: notificationTopic.topicArn, - 'accounts.$': '$[0].accounts', + parametersTableName: parametersTable.tableName, 'acceleratorVersion.$': '$[0].acceleratorVersion', }, resultPath: 'DISCARD', diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts index 1ef165997..5c59b5de2 100644 --- a/src/core/runtime/src/account-default-settings-step.ts +++ b/src/core/runtime/src/account-default-settings-step.ts @@ -3,7 +3,6 @@ import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { - StackOutput, getStackOutput, AWS_LANDING_ZONE_CLOUD_TRAIL_NAME, OUTPUT_LOG_ARCHIVE_ENCRYPTION_KEY_ARN, @@ -13,11 +12,12 @@ import { PutEventSelectorsRequest, UpdateTrailRequest } from 'aws-sdk/clients/cl import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadOutputs } from './utils/load-outputs'; +import { loadAccounts } from './utils/load-accounts'; interface AccountDefaultSettingsInput extends LoadConfigurationInput { assumeRoleName: string; - accounts: Account[]; outputTableName: string; + parametersTableName: string; } const dynamodb = new DynamoDB(); @@ -26,7 +26,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => { console.log('Setting account level defaults for all accounts in an organization ...'); console.log(JSON.stringify(input, null, 2)); - const { assumeRoleName, accounts, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; + const { assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName, parametersTableName } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ @@ -38,6 +38,7 @@ export const handler = async (input: AccountDefaultSettingsInput) => { const logAccountKey = acceleratorConfig.getMandatoryAccountKey('central-log'); const outputs = await loadOutputs(outputTableName, dynamodb); + const accounts = await loadAccounts(parametersTableName, dynamodb); const sts = new STS(); diff --git a/src/core/runtime/src/add-role-to-kms-key-step.ts b/src/core/runtime/src/add-role-to-kms-key-step.ts index a42724ee8..deaaf3a7a 100644 --- a/src/core/runtime/src/add-role-to-kms-key-step.ts +++ b/src/core/runtime/src/add-role-to-kms-key-step.ts @@ -1,19 +1,25 @@ import * as aws from 'aws-sdk'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { loadAccounts } from './utils/load-accounts'; interface AddRoleToKmsKeyInput { roleName: string; - accounts: Account[]; kmsKeyId: string; + parametersTableName: string; } +const dynamodb = new DynamoDB(); +const kms = new aws.KMS(); + export const handler = async (input: AddRoleToKmsKeyInput) => { console.log(`Adding roles to KMS key policy...`); console.log(JSON.stringify(input, null, 2)); - const { roleName, kmsKeyId, accounts } = input; + const { roleName, kmsKeyId, parametersTableName } = input; + + const accounts = await loadAccounts(parametersTableName, dynamodb); - const kms = new aws.KMS(); const getKeyPolicy = await kms .getKeyPolicy({ KeyId: kmsKeyId, diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index ba018ad57..46c8d960a 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -1,4 +1,4 @@ -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; @@ -6,12 +6,13 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifacts'; import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; import { loadOutputs } from './utils/load-outputs'; +import { getItemInput } from './utils/dynamodb-requests'; interface AddScpInput extends LoadConfigurationInput { acceleratorPrefix: string; - accounts: Account[]; - organizationalUnits: OrganizationalUnit[]; + accounts: ShortAccount[]; outputTableName: string; + parametersTableName: string; } const dynamodb = new DynamoDB(); @@ -23,11 +24,11 @@ export const handler = async (input: AddScpInput) => { const { acceleratorPrefix, accounts, - organizationalUnits, configRepositoryName, configFilePath, configCommitId, outputTableName, + parametersTableName, } = input; // Retrieve Configuration from Code Commit with specific commitId @@ -68,6 +69,11 @@ export const handler = async (input: AddScpInput) => { // Find roots to attach FullAWSAccess const rootIds = await scps.organizationRoots(); + const organizationsResponse = await dynamodb.getItem(getItemInput(parametersTableName, `organizations`)); + if (!organizationsResponse.Item) { + throw new Error(`No organizations found in DynamoDB "${parametersTableName}"`); + } + const organizationalUnits:OrganizationalUnit[] = JSON.parse(organizationsResponse.Item.value.S!); // Find Accelerator accounts and OUs to attach FullAWSAccess const acceleratorOuIds = organizationalUnits.map(ou => ou.ouId); const acceleratorAccountIds = accounts.map(a => a.id); diff --git a/src/core/runtime/src/associate-hosted-zones-step.ts b/src/core/runtime/src/associate-hosted-zones-step.ts index 8f4fa104f..afc2c2e4a 100644 --- a/src/core/runtime/src/associate-hosted-zones-step.ts +++ b/src/core/runtime/src/associate-hosted-zones-step.ts @@ -10,11 +10,12 @@ import { LoadConfigurationInput } from './load-configuration-step'; import { throttlingBackOff } from '@aws-accelerator/common/src/aws/backoff'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; import { loadOutputs } from './utils/load-outputs'; +import { loadAccounts } from './utils/load-accounts'; interface AssociateHostedZonesInput extends LoadConfigurationInput { - accounts: Account[]; assumeRoleName: string; outputTableName: string; + parametersTableName: string; } type ResolversOutputs = ResolversOutput[]; @@ -48,7 +49,9 @@ export const handler = async (input: AssociateHostedZonesInput) => { console.log(`Associating Hosted Zones with VPC...`); console.log(JSON.stringify(input, null, 2)); - const { configRepositoryName, accounts, assumeRoleName, configCommitId, configFilePath, outputTableName } = input; + const { configRepositoryName, assumeRoleName, configCommitId, configFilePath, outputTableName, parametersTableName } = input; + + const accounts = await loadAccounts(parametersTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/configuration/load-landing-zone-config.ts b/src/core/runtime/src/configuration/load-landing-zone-config.ts index 1bfcd497c..2eb3d8f3e 100644 --- a/src/core/runtime/src/configuration/load-landing-zone-config.ts +++ b/src/core/runtime/src/configuration/load-landing-zone-config.ts @@ -306,7 +306,7 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), regions: config['global-options']['supported-regions'], warnings, }; diff --git a/src/core/runtime/src/configuration/load-organizations-config.ts b/src/core/runtime/src/configuration/load-organizations-config.ts index be0415b84..aa21acbba 100644 --- a/src/core/runtime/src/configuration/load-organizations-config.ts +++ b/src/core/runtime/src/configuration/load-organizations-config.ts @@ -2,6 +2,7 @@ import * as org from 'aws-sdk/clients/organizations'; import { Organizations, OrganizationalUnit } from '@aws-accelerator/common/src/aws/organizations'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { STS } from '@aws-accelerator/common/src/aws/sts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; import { LoadConfigurationInput, @@ -227,7 +228,8 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), regions: config['global-options']['supported-regions'], warnings, installCloudFormationMasterRole, diff --git a/src/core/runtime/src/create-adconnector/create.ts b/src/core/runtime/src/create-adconnector/create.ts index 710ecb288..7055655cd 100644 --- a/src/core/runtime/src/create-adconnector/create.ts +++ b/src/core/runtime/src/create-adconnector/create.ts @@ -9,12 +9,13 @@ import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from '../load-configuration-step'; import { VpcOutputFinder } from '@aws-accelerator/common-outputs/src/vpc'; import { loadOutputs } from '../utils/load-outputs'; +import { loadAccounts } from '../utils/load-accounts'; const VALID_STATUSES: string[] = ['Requested', 'Creating', 'Created', 'Active', 'Inoperable', 'Impaired', 'Restoring']; interface AdConnectorInput extends LoadConfigurationInput { acceleratorPrefix: string; - accounts: Account[]; + parametersTableName: string; assumeRoleName: string; configRepositoryName: string; configFilePath: string; @@ -38,7 +39,7 @@ export const handler = async (input: AdConnectorInput) => { const { acceleratorPrefix, - accounts, + parametersTableName, assumeRoleName, configRepositoryName, configFilePath, @@ -46,6 +47,7 @@ export const handler = async (input: AdConnectorInput) => { outputTableName, } = input; + const accounts = await loadAccounts(parametersTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ repositoryName: configRepositoryName, diff --git a/src/core/runtime/src/detach-quarantine-scp.ts b/src/core/runtime/src/detach-quarantine-scp.ts index 7e28a83c6..ab5bc25b9 100644 --- a/src/core/runtime/src/detach-quarantine-scp.ts +++ b/src/core/runtime/src/detach-quarantine-scp.ts @@ -1,18 +1,21 @@ -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; +import { loadAccounts } from './utils/load-accounts'; interface DetachQuarantineScpInput { - accounts: Account[]; acceleratorPrefix: string; + parametersTableName: string; } const organizations = new Organizations(); +const dynamodb = new DynamoDB(); export const handler = async (input: DetachQuarantineScpInput): Promise => { console.log(`Creating account using Organizations...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, accounts } = input; + const { acceleratorPrefix, parametersTableName} = input; + const accounts = await loadAccounts(parametersTableName, dynamodb); const policyName = ServiceControlPolicy.createQuarantineScpName({ acceleratorPrefix }); diff --git a/src/core/runtime/src/enable-directory-sharing-step.ts b/src/core/runtime/src/enable-directory-sharing-step.ts index b7d9c030b..ad780d710 100644 --- a/src/core/runtime/src/enable-directory-sharing-step.ts +++ b/src/core/runtime/src/enable-directory-sharing-step.ts @@ -1,15 +1,16 @@ import { DirectoryService } from '@aws-accelerator/common/src/aws/directory-service'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; -import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; +import { getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { MadOutput } from '@aws-accelerator/common-outputs/src/mad'; import { STS } from '@aws-accelerator/common/src/aws/sts'; -import { StackOutput, getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadOutputs } from './utils/load-outputs'; +import { loadAccounts } from './utils/load-accounts'; interface ShareDirectoryInput extends LoadConfigurationInput { - accounts: Account[]; + parametersTableName: string; assumeRoleName: string; outputTableName: string; } @@ -20,8 +21,9 @@ export const handler = async (input: ShareDirectoryInput) => { console.log(`Sharing MAD to another account ...`); console.log(JSON.stringify(input, null, 2)); - const { accounts, assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; - + const { parametersTableName, assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; + + const accounts = await loadAccounts(parametersTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ repositoryName: configRepositoryName, diff --git a/src/core/runtime/src/enable-trusted-access-for-services-step.ts b/src/core/runtime/src/enable-trusted-access-for-services-step.ts index e96cc5d99..27071c0c0 100644 --- a/src/core/runtime/src/enable-trusted-access-for-services-step.ts +++ b/src/core/runtime/src/enable-trusted-access-for-services-step.ts @@ -2,19 +2,22 @@ import * as aws from 'aws-sdk'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { FMS } from '@aws-accelerator/common/src/aws/fms'; import { IAM } from '@aws-accelerator/common/src/aws/iam'; -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; +import { loadAccounts } from './utils/load-accounts'; interface EnableTrustedAccessForServicesInput extends LoadConfigurationInput { - accounts: Account[]; + parametersTableName: string; } +const dynamodb = new DynamoDB(); + export const handler = async (input: EnableTrustedAccessForServicesInput) => { console.log(`Enable Trusted Access for AWS services within the organization ...`); console.log(JSON.stringify(input, null, 2)); - const { accounts, configRepositoryName, configFilePath, configCommitId } = input; + const { parametersTableName, configRepositoryName, configFilePath, configCommitId } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -24,7 +27,7 @@ export const handler = async (input: EnableTrustedAccessForServicesInput) => { }); const securityAccountKey = config['global-options']['central-security-services'].account; - + const accounts = await loadAccounts(parametersTableName, dynamodb); const securityAccount = accounts.find(a => a.key === securityAccountKey); if (!securityAccount) { console.warn('Cannot find account with type security'); diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index 5458f15db..70c757b49 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,33 +1,47 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step'; +import { Account, ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; +import { + LoadConfigurationInput, + ConfigurationAccount, +} from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests'; +import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; -export interface LoadAccountsInput { +export interface LoadAccountsInput extends LoadConfigurationInput{ accountsItemsCountId: string; parametersTableName: string; itemId: string; - configuration: LoadConfigurationOutput; + accounts: ConfigurationAccount[]; + regions: string[]; } export interface LoadAccountsOutput { - organizationalUnits: ConfigurationOrganizationalUnit[]; - accounts: Account[]; + accounts: ShortAccount[]; regions: string[]; } const dynamoDB = new DynamoDB(); +const organizations = new Organizations(); export const handler = async (input: LoadAccountsInput): Promise => { console.log(`Loading accounts...`); console.log(JSON.stringify(input, null, 2)); - const { parametersTableName, configuration, itemId, accountsItemsCountId } = input; + const { parametersTableName, itemId, accountsItemsCountId, configRepositoryName, configCommitId, configFilePath } = input; + - // The first step is to load all the execution roles - const organizations = new Organizations(); + // Retrieve Configuration from Code Commit with specific commitId + const config = await loadAcceleratorConfig({ + repositoryName: configRepositoryName, + filePath: configFilePath, + commitId: configCommitId, + }); + const ignoredOus = config['global-options']['ignored-ous'] || []; + // First load mandatory accounts configuration + const mandatoryAccounts = config.getMandatoryAccountConfigs(); + const mandatoryAccountKeys = mandatoryAccounts.map(([accountKey, _]) => accountKey); const organizationAccounts = await organizations.listAccounts(); const activeAccounts = organizationAccounts.filter(account => account.Status === 'ACTIVE'); @@ -38,44 +52,41 @@ export const handler = async (input: LoadAccountsInput): Promise { - return equalIgnoreCase(a.Email!, accountConfig.emailAddress); + return equalIgnoreCase(a.Email!, accountConfig.email); }); - // TODO Removing "landingZoneAccountType" check for mandatory account. Can be replaced with "accountName" after proper testing - // if (accountConfig.landingZoneAccountType === 'primary') { - // // Only filter on the email address if we are dealing with the master account - // organizationAccount = organizationAccounts.find(a => { - // return a.Email === accountConfig.emailAddress; - // }); - // } else { - // organizationAccount = organizationAccounts.find(a => { - // return a.Name === accountConfig.accountName && a.Email === accountConfig.emailAddress; - // }); - // } + // Find the organizational account used by this + const organizationalUnitName = accountConfig.ou; + + if (ignoredOus.includes(organizationalUnitName)) { + console.warn(`Account ${accountKey} found under ignored OU "${organizationalUnitName}"`); + continue; + } + if (!organizationAccount) { - if (!accountConfig.isMandatoryAccount) { + if (!mandatoryAccountKeys.includes(accountKey)) { console.warn( - `Cannot find non mandatory account with name "${accountConfig.accountName}" and email "${accountConfig.emailAddress}"`, + `Cannot find non mandatory account with name "${accountConfig['account-name']}" and email "${accountConfig.email}"`, ); continue; } throw new Error( - `Cannot find account with name "${accountConfig.accountName}" and email "${accountConfig.emailAddress}"`, + `Cannot find account with name "${accountConfig['account-name']}" and email "${accountConfig.email}"`, ); } accounts.push({ - key: accountConfig.accountKey, + key: accountKey, id: organizationAccount.Id!, arn: organizationAccount.Arn!, name: organizationAccount.Name!, email: organizationAccount.Email!, - ou: accountConfig.organizationalUnit, - type: accountConfig.landingZoneAccountType, - ouPath: accountConfig.ouPath, + ou: accountConfig.ou, + ouPath: accountConfig['ou-path'], }); } @@ -100,9 +111,12 @@ export const handler = async (input: LoadAccountsInput): Promise ({ + id: acc.id, + key: acc.key + })); return { - ...configuration, - accounts, + ...input, + accounts: shortAccounts, }; }; diff --git a/src/core/runtime/src/load-limits-step.ts b/src/core/runtime/src/load-limits-step.ts index 4ad02539b..8b3ea9c29 100644 --- a/src/core/runtime/src/load-limits-step.ts +++ b/src/core/runtime/src/load-limits-step.ts @@ -1,16 +1,16 @@ import { ServiceQuotas } from '@aws-accelerator/common/src/aws/service-quotas'; -import { Account, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; +import { ShortAccount, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { getUpdateItemInput } from './utils/dynamodb-requests'; +import { loadAccounts } from './utils/load-accounts'; export interface LoadLimitsInput extends LoadConfigurationInput { parametersTableName: string; itemId: string; - accounts: Account[]; assumeRoleName: string; } @@ -53,7 +53,7 @@ const LIMITS: { [limitKey: string]: LimitCode } = { }, }; -const dynamoDB = new DynamoDB(); +const dynamodb = new DynamoDB(); export const handler = async (input: LoadLimitsInput) => { console.log(`Loading limits...`); @@ -63,12 +63,13 @@ export const handler = async (input: LoadLimitsInput) => { configRepositoryName, configFilePath, parametersTableName, - accounts, assumeRoleName, configCommitId, itemId, } = input; + const accounts = await loadAccounts(parametersTableName, dynamodb); + // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ repositoryName: configRepositoryName, @@ -160,5 +161,5 @@ export const handler = async (input: LoadLimitsInput) => { } // Store the limits in the dynamodb - await dynamoDB.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(limits, null, 2))); + await dynamodb.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(limits, null, 2))); }; diff --git a/src/core/runtime/src/notify-statemachine-success.ts b/src/core/runtime/src/notify-statemachine-success.ts index 7aebb3a45..338d65571 100644 --- a/src/core/runtime/src/notify-statemachine-success.ts +++ b/src/core/runtime/src/notify-statemachine-success.ts @@ -1,21 +1,25 @@ import { SNS } from '@aws-accelerator/common/src/aws/sns'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { loadAccounts } from './utils/load-accounts'; const MAX_SNS_PUBLISH_CHAR = 255500; const AVG_CHARS_PER_ACCOUNT = 100; interface NotifySuccessInput { notificationTopicArn: string; - accounts: Account[]; + parametersTableName: string; acceleratorVersion?: string; } const sns = new SNS(); +const dynamodb = new DynamoDB(); export const handler = async (input: NotifySuccessInput): Promise => { console.log('State Machine Execution Success...'); console.log(JSON.stringify(input, null, 2)); - const { accounts, acceleratorVersion } = input; + const { acceleratorVersion, parametersTableName } = input; + const accounts = await loadAccounts(parametersTableName, dynamodb); const responseAccounts = accounts.map(acc => ({ key: acc.key, id: acc.id, diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index 1abf7e454..e814763f2 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -1,4 +1,4 @@ -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; @@ -7,7 +7,7 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; export interface StoreStackOutputInput { acceleratorPrefix: string; assumeRoleName: string; - account: Account; + account: ShortAccount; region: string; outputsTable: string; phaseNumber: number; diff --git a/src/core/runtime/src/utils/load-accounts.ts b/src/core/runtime/src/utils/load-accounts.ts new file mode 100644 index 000000000..2e491b2a1 --- /dev/null +++ b/src/core/runtime/src/utils/load-accounts.ts @@ -0,0 +1,24 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; + +export async function loadAccounts(tableName: string, client: DynamoDB): Promise { + let index = 0; + const accounts: Account[] = []; + while (true) { + const itemsInput = { + TableName: tableName, + Key: { id: { S: `accounts/${index}` } }, + }; + const item = await new DynamoDB().getItem(itemsInput); + if (index === 0 && !item.Item) { + throw new Error(`Cannot find parameter with ID "accounts"`); + } + + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; + } + return accounts; +} \ No newline at end of file diff --git a/src/lib/common-outputs/src/accounts.ts b/src/lib/common-outputs/src/accounts.ts index 664e52c3b..f818139a0 100644 --- a/src/lib/common-outputs/src/accounts.ts +++ b/src/lib/common-outputs/src/accounts.ts @@ -13,7 +13,12 @@ export interface Account { ouPath?: string; } -export function getAccountId(accounts: Account[], accountKey: string): string | undefined { +export interface ShortAccount { + key: string; + id: string; +} + +export function getAccountId(accounts: Account[] | ShortAccount[], accountKey: string): string | undefined { const account = accounts.find(a => a.key === accountKey); if (!account) { console.warn(`Cannot find account with key "${accountKey}"`); From bb0a283b33cfa0a40bf027a174cc7a18937823f8 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 17:35:40 +0530 Subject: [PATCH 02/20] Prettier --- src/core/cdk/src/initial-setup.ts | 1 + .../src/account-default-settings-step.ts | 9 ++++- .../runtime/src/add-role-to-kms-key-step.ts | 2 +- src/core/runtime/src/add-scp-step.ts | 2 +- .../src/associate-hosted-zones-step.ts | 9 ++++- .../configuration/load-landing-zone-config.ts | 2 +- .../load-organizations-config.ts | 2 +- src/core/runtime/src/detach-quarantine-scp.ts | 2 +- .../src/enable-directory-sharing-step.ts | 11 +++++- src/core/runtime/src/load-accounts-step.ts | 19 ++++++---- src/core/runtime/src/load-limits-step.ts | 9 +---- src/core/runtime/src/utils/load-accounts.ts | 38 +++++++++---------- 12 files changed, 62 insertions(+), 44 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 3fc464cba..e3c85df67 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -334,6 +334,7 @@ export namespace InitialSetup { 'baseline.$': '$.configuration.baseline', 'regions.$': '$.configuration.regions', 'accounts.$': '$.configuration.accounts', + 'configRootFilePath.$': '$.configuration.configRootFilePath', }, resultPath: '$', }); diff --git a/src/core/runtime/src/account-default-settings-step.ts b/src/core/runtime/src/account-default-settings-step.ts index 5c59b5de2..d3036ddfc 100644 --- a/src/core/runtime/src/account-default-settings-step.ts +++ b/src/core/runtime/src/account-default-settings-step.ts @@ -26,7 +26,14 @@ export const handler = async (input: AccountDefaultSettingsInput) => { console.log('Setting account level defaults for all accounts in an organization ...'); console.log(JSON.stringify(input, null, 2)); - const { assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName, parametersTableName } = input; + const { + assumeRoleName, + configRepositoryName, + configFilePath, + configCommitId, + outputTableName, + parametersTableName, + } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/add-role-to-kms-key-step.ts b/src/core/runtime/src/add-role-to-kms-key-step.ts index deaaf3a7a..5f0766232 100644 --- a/src/core/runtime/src/add-role-to-kms-key-step.ts +++ b/src/core/runtime/src/add-role-to-kms-key-step.ts @@ -17,7 +17,7 @@ export const handler = async (input: AddRoleToKmsKeyInput) => { console.log(JSON.stringify(input, null, 2)); const { roleName, kmsKeyId, parametersTableName } = input; - + const accounts = await loadAccounts(parametersTableName, dynamodb); const getKeyPolicy = await kms diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index 46c8d960a..83997b78f 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -73,7 +73,7 @@ export const handler = async (input: AddScpInput) => { if (!organizationsResponse.Item) { throw new Error(`No organizations found in DynamoDB "${parametersTableName}"`); } - const organizationalUnits:OrganizationalUnit[] = JSON.parse(organizationsResponse.Item.value.S!); + const organizationalUnits: OrganizationalUnit[] = JSON.parse(organizationsResponse.Item.value.S!); // Find Accelerator accounts and OUs to attach FullAWSAccess const acceleratorOuIds = organizationalUnits.map(ou => ou.ouId); const acceleratorAccountIds = accounts.map(a => a.id); diff --git a/src/core/runtime/src/associate-hosted-zones-step.ts b/src/core/runtime/src/associate-hosted-zones-step.ts index afc2c2e4a..9760b7dd5 100644 --- a/src/core/runtime/src/associate-hosted-zones-step.ts +++ b/src/core/runtime/src/associate-hosted-zones-step.ts @@ -49,7 +49,14 @@ export const handler = async (input: AssociateHostedZonesInput) => { console.log(`Associating Hosted Zones with VPC...`); console.log(JSON.stringify(input, null, 2)); - const { configRepositoryName, assumeRoleName, configCommitId, configFilePath, outputTableName, parametersTableName } = input; + const { + configRepositoryName, + assumeRoleName, + configCommitId, + configFilePath, + outputTableName, + parametersTableName, + } = input; const accounts = await loadAccounts(parametersTableName, dynamodb); diff --git a/src/core/runtime/src/configuration/load-landing-zone-config.ts b/src/core/runtime/src/configuration/load-landing-zone-config.ts index 2eb3d8f3e..1f6467556 100644 --- a/src/core/runtime/src/configuration/load-landing-zone-config.ts +++ b/src/core/runtime/src/configuration/load-landing-zone-config.ts @@ -306,7 +306,7 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), + accounts: configurationAccounts.filter(acc => !acc.accountId), regions: config['global-options']['supported-regions'], warnings, }; diff --git a/src/core/runtime/src/configuration/load-organizations-config.ts b/src/core/runtime/src/configuration/load-organizations-config.ts index aa21acbba..2ef54c20f 100644 --- a/src/core/runtime/src/configuration/load-organizations-config.ts +++ b/src/core/runtime/src/configuration/load-organizations-config.ts @@ -229,7 +229,7 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), + accounts: configurationAccounts.filter(acc => !acc.accountId), regions: config['global-options']['supported-regions'], warnings, installCloudFormationMasterRole, diff --git a/src/core/runtime/src/detach-quarantine-scp.ts b/src/core/runtime/src/detach-quarantine-scp.ts index ab5bc25b9..8d606a6c3 100644 --- a/src/core/runtime/src/detach-quarantine-scp.ts +++ b/src/core/runtime/src/detach-quarantine-scp.ts @@ -14,7 +14,7 @@ export const handler = async (input: DetachQuarantineScpInput): Promise console.log(`Creating account using Organizations...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, parametersTableName} = input; + const { acceleratorPrefix, parametersTableName } = input; const accounts = await loadAccounts(parametersTableName, dynamodb); const policyName = ServiceControlPolicy.createQuarantineScpName({ acceleratorPrefix }); diff --git a/src/core/runtime/src/enable-directory-sharing-step.ts b/src/core/runtime/src/enable-directory-sharing-step.ts index ad780d710..4ba5d90fb 100644 --- a/src/core/runtime/src/enable-directory-sharing-step.ts +++ b/src/core/runtime/src/enable-directory-sharing-step.ts @@ -21,8 +21,15 @@ export const handler = async (input: ShareDirectoryInput) => { console.log(`Sharing MAD to another account ...`); console.log(JSON.stringify(input, null, 2)); - const { parametersTableName, assumeRoleName, configRepositoryName, configFilePath, configCommitId, outputTableName } = input; - + const { + parametersTableName, + assumeRoleName, + configRepositoryName, + configFilePath, + configCommitId, + outputTableName, + } = input; + const accounts = await loadAccounts(parametersTableName, dynamodb); // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index 70c757b49..b784d3b03 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,15 +1,12 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { Account, ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; -import { - LoadConfigurationInput, - ConfigurationAccount, -} from './load-configuration-step'; +import { LoadConfigurationInput, ConfigurationAccount } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; -export interface LoadAccountsInput extends LoadConfigurationInput{ +export interface LoadAccountsInput extends LoadConfigurationInput { accountsItemsCountId: string; parametersTableName: string; itemId: string; @@ -29,8 +26,14 @@ export const handler = async (input: LoadAccountsInput): Promise ({ id: acc.id, - key: acc.key + key: acc.key, })); return { ...input, diff --git a/src/core/runtime/src/load-limits-step.ts b/src/core/runtime/src/load-limits-step.ts index 8b3ea9c29..d465d7838 100644 --- a/src/core/runtime/src/load-limits-step.ts +++ b/src/core/runtime/src/load-limits-step.ts @@ -59,14 +59,7 @@ export const handler = async (input: LoadLimitsInput) => { console.log(`Loading limits...`); console.log(JSON.stringify(input, null, 2)); - const { - configRepositoryName, - configFilePath, - parametersTableName, - assumeRoleName, - configCommitId, - itemId, - } = input; + const { configRepositoryName, configFilePath, parametersTableName, assumeRoleName, configCommitId, itemId } = input; const accounts = await loadAccounts(parametersTableName, dynamodb); diff --git a/src/core/runtime/src/utils/load-accounts.ts b/src/core/runtime/src/utils/load-accounts.ts index 2e491b2a1..99db26ca5 100644 --- a/src/core/runtime/src/utils/load-accounts.ts +++ b/src/core/runtime/src/utils/load-accounts.ts @@ -2,23 +2,23 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; export async function loadAccounts(tableName: string, client: DynamoDB): Promise { - let index = 0; - const accounts: Account[] = []; - while (true) { - const itemsInput = { - TableName: tableName, - Key: { id: { S: `accounts/${index}` } }, - }; - const item = await new DynamoDB().getItem(itemsInput); - if (index === 0 && !item.Item) { - throw new Error(`Cannot find parameter with ID "accounts"`); - } - - if (!item.Item) { - break; - } - accounts.push(...JSON.parse(item.Item.value.S!)); - index++; + let index = 0; + const accounts: Account[] = []; + while (true) { + const itemsInput = { + TableName: tableName, + Key: { id: { S: `accounts/${index}` } }, + }; + const item = await new DynamoDB().getItem(itemsInput); + if (index === 0 && !item.Item) { + throw new Error(`Cannot find parameter with ID "accounts"`); } - return accounts; -} \ No newline at end of file + + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; + } + return accounts; +} From 227e1c72fb9ff70423ee9d4e0ee8a2fc34c0d73f Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 18:26:26 +0530 Subject: [PATCH 03/20] Fixing Add SCPs --- src/core/runtime/src/add-scp-step.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index 83997b78f..d87aacc40 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -1,4 +1,3 @@ -import { ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; @@ -7,10 +6,10 @@ import { ArtifactOutputFinder } from '@aws-accelerator/common-outputs/src/artifa import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; import { loadOutputs } from './utils/load-outputs'; import { getItemInput } from './utils/dynamodb-requests'; +import { loadAccounts } from './utils/load-accounts'; interface AddScpInput extends LoadConfigurationInput { acceleratorPrefix: string; - accounts: ShortAccount[]; outputTableName: string; parametersTableName: string; } @@ -23,7 +22,6 @@ export const handler = async (input: AddScpInput) => { const { acceleratorPrefix, - accounts, configRepositoryName, configFilePath, configCommitId, @@ -41,6 +39,7 @@ export const handler = async (input: AddScpInput) => { const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole); const outputs = await loadOutputs(outputTableName, dynamodb); + const accounts = await loadAccounts(parametersTableName, dynamodb); // Find the SCP artifact output const artifactOutput = ArtifactOutputFinder.findOneByName({ @@ -68,8 +67,8 @@ export const handler = async (input: AddScpInput) => { // Find roots to attach FullAWSAccess const rootIds = await scps.organizationRoots(); - - const organizationsResponse = await dynamodb.getItem(getItemInput(parametersTableName, `organizations`)); + + const organizationsResponse = await dynamodb.getItem(getItemInput(parametersTableName, 'organizations')); if (!organizationsResponse.Item) { throw new Error(`No organizations found in DynamoDB "${parametersTableName}"`); } From 333bf28b81568c734d5da69001091b414cac4623 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 18:30:07 +0530 Subject: [PATCH 04/20] Having only accountIds list in SM input --- src/core/runtime/src/add-scp-step.ts | 2 +- .../src/create-config-recorder/create.ts | 25 ++++++---- .../runtime/src/delete-default-vpc/delete.ts | 5 +- src/core/runtime/src/load-accounts-step.ts | 11 ++--- src/core/runtime/src/load-limits-step.ts | 2 +- .../runtime/src/store-stack-output-step.ts | 46 +++++++++++++++---- 6 files changed, 59 insertions(+), 32 deletions(-) diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index d87aacc40..27517b646 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -67,7 +67,7 @@ export const handler = async (input: AddScpInput) => { // Find roots to attach FullAWSAccess const rootIds = await scps.organizationRoots(); - + const organizationsResponse = await dynamodb.getItem(getItemInput(parametersTableName, 'organizations')); if (!organizationsResponse.Item) { throw new Error(`No organizations found in DynamoDB "${parametersTableName}"`); diff --git a/src/core/runtime/src/create-config-recorder/create.ts b/src/core/runtime/src/create-config-recorder/create.ts index 761ed709c..c2518f1d2 100644 --- a/src/core/runtime/src/create-config-recorder/create.ts +++ b/src/core/runtime/src/create-config-recorder/create.ts @@ -1,9 +1,9 @@ import { ConfigService } from '@aws-accelerator/common/src/aws/configservice'; import { ConfigurationRecorder } from 'aws-sdk/clients/configservice'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { getStackJsonOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { LoadConfigurationInput } from '../load-configuration-step'; -import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { createConfigRecorderName, createAggregatorName } from '@aws-accelerator/common-outputs/src/config'; @@ -11,7 +11,7 @@ import { IamRoleOutputFinder } from '@aws-accelerator/common-outputs/src/iam-rol import { loadOutputs } from '../utils/load-outputs'; interface ConfigServiceInput extends LoadConfigurationInput { - account: Account; + accountId: string; assumeRoleName: string; acceleratorPrefix: string; outputTableName: string; @@ -36,12 +36,13 @@ const CustomErrorMessage = [ const sts = new STS(); const dynamodb = new DynamoDB(); +const organizations = new Organizations(); export const handler = async (input: ConfigServiceInput): Promise => { console.log(`Enable Config Recorder in account ...`); console.log(JSON.stringify(input, null, 2)); const { - account, + accountId, assumeRoleName, configRepositoryName, configFilePath, @@ -59,7 +60,11 @@ export const handler = async (input: ConfigServiceInput): Promise => { commitId: configCommitId, }); - const accountId = account.id; + const awsAccount = await organizations.getAccount(accountId); + const configAccount = acceleratorConfig + .getAccountConfigs() + .find(([_, accountConfig]) => accountConfig.email === awsAccount?.Email); + const accountKey = configAccount?.[0]!; const masterAccountKey = acceleratorConfig.getMandatoryAccountKey('master'); const centralSecurityRegion = acceleratorConfig['global-options']['central-security-services'].region; const supportedRegions = acceleratorConfig['global-options']['supported-regions']; @@ -82,12 +87,12 @@ export const handler = async (input: ConfigServiceInput): Promise => { const configRecorderRole = IamRoleOutputFinder.tryFindOneByName({ outputs, - accountKey: account.key, + accountKey, roleKey: 'ConfigRecorderRole', }); if (!configRecorderRole) { - errors.push(`${accountId}:: No ConfigRecorderRole created in Account "${account.key}"`); + errors.push(`${accountId}:: No ConfigRecorderRole created in Account "${accountKey}"`); return errors; } @@ -140,7 +145,7 @@ export const handler = async (input: ConfigServiceInput): Promise => { ); errors.push(...createChannel); - console.log(`${account.id}::${region}:: Enabling Config Recorder`); + console.log(`${accountId}::${region}:: Enabling Config Recorder`); const enableConfig = await enableConfigRecorder(configService, accountId, region, acceleratorRecorderName); errors.push(...enableConfig); } catch (error) { @@ -153,14 +158,14 @@ export const handler = async (input: ConfigServiceInput): Promise => { } } - if (account.key === masterAccountKey) { + if (accountKey === masterAccountKey) { const configAggregatorRole = IamRoleOutputFinder.tryFindOneByName({ outputs, - accountKey: account.key, + accountKey, roleKey: 'ConfigAggregatorRole', }); if (!configAggregatorRole) { - errors.push(`${accountId}:: No Aggregaror Role created in Master Account ${account.key}`); + errors.push(`${accountId}:: No Aggregaror Role created in Master Account ${accountKey}`); } else { const configService = new ConfigService(credentials, centralSecurityRegion); const enableAggregator = await createAggregator( diff --git a/src/core/runtime/src/delete-default-vpc/delete.ts b/src/core/runtime/src/delete-default-vpc/delete.ts index 2f538c2e2..fc914076c 100644 --- a/src/core/runtime/src/delete-default-vpc/delete.ts +++ b/src/core/runtime/src/delete-default-vpc/delete.ts @@ -5,7 +5,7 @@ import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; interface DeleteVPCInput extends LoadConfigurationInput { - account: Account; + accountId: string; assumeRoleName: string; } @@ -24,7 +24,7 @@ const sts = new STS(); export const handler = async (input: DeleteVPCInput): Promise => { console.log(`Deleting Default VPC in account ...`); console.log(JSON.stringify(input, null, 2)); - const { account, assumeRoleName, configRepositoryName, configFilePath, configCommitId } = input; + const { accountId, assumeRoleName, configRepositoryName, configFilePath, configCommitId } = input; // Retrieve Configuration from Code Commit with specific commitId const acceleratorConfig = await loadAcceleratorConfig({ @@ -32,7 +32,6 @@ export const handler = async (input: DeleteVPCInput): Promise => { filePath: configFilePath, commitId: configCommitId, }); - const accountId = account.id; const supportedRegions = acceleratorConfig['global-options']['supported-regions']; const excludeRegions = acceleratorConfig['global-options']['keep-default-vpc-regions']; const regions = supportedRegions.filter(r => !excludeRegions.includes(r)); diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index b784d3b03..009f5e0cb 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,5 +1,5 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; -import { Account, ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { LoadConfigurationInput, ConfigurationAccount } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; @@ -15,7 +15,7 @@ export interface LoadAccountsInput extends LoadConfigurationInput { } export interface LoadAccountsOutput { - accounts: ShortAccount[]; + accounts: string[]; regions: string[]; } @@ -114,12 +114,9 @@ export const handler = async (input: LoadAccountsInput): Promise ({ - id: acc.id, - key: acc.key, - })); + const accountIds: string[] = accounts.map(acc => acc.id); return { ...input, - accounts: shortAccounts, + accounts: accountIds, }; }; diff --git a/src/core/runtime/src/load-limits-step.ts b/src/core/runtime/src/load-limits-step.ts index d465d7838..451196b52 100644 --- a/src/core/runtime/src/load-limits-step.ts +++ b/src/core/runtime/src/load-limits-step.ts @@ -1,5 +1,5 @@ import { ServiceQuotas } from '@aws-accelerator/common/src/aws/service-quotas'; -import { ShortAccount, getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; +import { getAccountId } from '@aws-accelerator/common-outputs/src/accounts'; import { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index e814763f2..8907eb38c 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -1,13 +1,15 @@ -import { ShortAccount } from '@aws-accelerator/common-outputs/src/accounts'; import { STS } from '@aws-accelerator/common/src/aws/sts'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { CloudFormation } from '@aws-accelerator/common/src/aws/cloudformation'; import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; +import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; +import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; +import { LoadConfigurationInput } from './load-configuration-step'; -export interface StoreStackOutputInput { +export interface StoreStackOutputInput extends LoadConfigurationInput { acceleratorPrefix: string; assumeRoleName: string; - account: ShortAccount; + accountId: string; region: string; outputsTable: string; phaseNumber: number; @@ -15,17 +17,41 @@ export interface StoreStackOutputInput { const sts = new STS(); const dynamodb = new DynamoDB(); +const organizations = new Organizations(); export const handler = async (input: StoreStackOutputInput) => { console.log(`Storing stack output...`); console.log(JSON.stringify(input, null, 2)); - const { acceleratorPrefix, assumeRoleName, account, region, outputsTable, phaseNumber } = input; - const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); + const { + acceleratorPrefix, + assumeRoleName, + accountId, + region, + outputsTable, + phaseNumber, + configCommitId, + configFilePath, + configRepositoryName, + } = input; + const credentials = await sts.getCredentialsForAccountAndRole(accountId, assumeRoleName); const cfn = new CloudFormation(credentials, region); const stacks = cfn.listStacksGenerator({ StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], }); + // Retrieve Configuration from Code Commit with specific commitId + const acceleratorConfig = await loadAcceleratorConfig({ + repositoryName: configRepositoryName, + filePath: configFilePath, + commitId: configCommitId, + }); + + const awsAccount = await organizations.getAccount(accountId); + const configAccount = acceleratorConfig + .getAccountConfigs() + .find(([_, accountConfig]) => accountConfig.email === awsAccount?.Email); + const accountKey = configAccount?.[0]!; + const outputs: StackOutput[] = []; for await (const summary of stacks) { if (!summary.StackName.match(`${acceleratorPrefix}(.*)-Phase${phaseNumber}`)) { @@ -46,7 +72,7 @@ export const handler = async (input: StoreStackOutputInput) => { console.debug(`Storing outputs for stack with name "${summary.StackName}"`); stack.Outputs?.forEach(output => outputs.push({ - accountKey: account.key, + accountKey, outputKey: `${output.OutputKey}sjkdh`, outputValue: output.OutputValue, outputDescription: output.Description, @@ -56,11 +82,11 @@ export const handler = async (input: StoreStackOutputInput) => { ); } if (outputs.length === 0) { - console.warn(`No outputs found for Account: ${account.key} and Region: ${region}`); + console.warn(`No outputs found for Account: ${accountKey} and Region: ${region}`); await dynamodb.deleteItem({ TableName: outputsTable, Key: { - id: { S: `${account.key}-${region}-${phaseNumber}` }, + id: { S: `${accountKey}-${region}-${phaseNumber}` }, }, }); return { @@ -69,8 +95,8 @@ export const handler = async (input: StoreStackOutputInput) => { } await dynamodb.putItem({ Item: { - id: { S: `${account.key}-${region}-${phaseNumber}` }, - accountKey: { S: account.key }, + id: { S: `${accountKey}-${region}-${phaseNumber}` }, + accountKey: { S: accountKey }, region: { S: region }, phase: { N: `${phaseNumber}` }, outputValue: { S: JSON.stringify(outputs) }, From 094c387a53ca92fd14558717cf8fddc2aa21120d Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 19:22:29 +0530 Subject: [PATCH 05/20] Passing only AccountId to Run Accross Accounts --- src/core/cdk/src/tasks/run-across-accounts-task.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/cdk/src/tasks/run-across-accounts-task.ts b/src/core/cdk/src/tasks/run-across-accounts-task.ts index 54710f26e..9682410db 100644 --- a/src/core/cdk/src/tasks/run-across-accounts-task.ts +++ b/src/core/cdk/src/tasks/run-across-accounts-task.ts @@ -66,9 +66,9 @@ export class RunAcrossAccountsTask extends sfn.StateMachineFragment { const mapTask = new sfn.Map(this, `${name} Map`, { itemsPath: '$.accounts', resultPath: '$.errors', - maxConcurrency: 5, + maxConcurrency: 10, parameters: { - 'account.$': '$$.Map.Item.Value', + 'accountId.$': '$$.Map.Item.Value', assumeRoleName, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', From 13a7465ccbf0ff11c3dc5e8f27aa0697302c2d37 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 20:31:08 +0530 Subject: [PATCH 06/20] create Execution role fix --- src/core/cdk/src/initial-setup.ts | 2 +- src/core/cdk/src/tasks/run-across-accounts-task.ts | 2 +- src/core/cdk/src/tasks/store-outputs-task.ts | 4 ++-- src/core/runtime/src/delete-default-vpc/delete.ts | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index e3c85df67..45ce17f37 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -404,7 +404,7 @@ export namespace InitialSetup { s3BucketName: installRoleTemplate.s3BucketName, s3ObjectKey: installRoleTemplate.s3ObjectKey, }, - 'instanceAccounts.$': '$.accounts[*].id', + 'instanceAccounts.$': '$.accounts', instanceRegions: [stack.region], }, }), diff --git a/src/core/cdk/src/tasks/run-across-accounts-task.ts b/src/core/cdk/src/tasks/run-across-accounts-task.ts index 9682410db..fd9c5dc80 100644 --- a/src/core/cdk/src/tasks/run-across-accounts-task.ts +++ b/src/core/cdk/src/tasks/run-across-accounts-task.ts @@ -52,7 +52,7 @@ export class RunAcrossAccountsTask extends sfn.StateMachineFragment { handler: `${lambdaPath}.run`, }, functionPayload: { - 'account.$': '$.account', + 'accountId.$': '$.accountId', assumeRoleName, 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 2eed40e7f..9313ae760 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -35,7 +35,7 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { resultPath: 'DISCARD', maxConcurrency: 10, parameters: { - 'account.$': '$$.Map.Item.Value', + 'accountId.$': '$$.Map.Item.Value', 'regions.$': '$.regions', 'acceleratorPrefix.$': '$.acceleratorPrefix', 'assumeRoleName.$': '$.assumeRoleName', @@ -49,7 +49,7 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { resultPath: 'DISCARD', maxConcurrency: 10, parameters: { - 'account.$': '$.account', + 'accountId.$': '$.accountId', 'region.$': '$$.Map.Item.Value', 'acceleratorPrefix.$': '$.acceleratorPrefix', 'assumeRoleName.$': '$.assumeRoleName', diff --git a/src/core/runtime/src/delete-default-vpc/delete.ts b/src/core/runtime/src/delete-default-vpc/delete.ts index fc914076c..b9e38206f 100644 --- a/src/core/runtime/src/delete-default-vpc/delete.ts +++ b/src/core/runtime/src/delete-default-vpc/delete.ts @@ -38,7 +38,7 @@ export const handler = async (input: DeleteVPCInput): Promise => { console.log(`${accountId}: Excluding Deletion of Default VPC for regions from account "${accountId}"...`); console.log(`${accountId}: ${JSON.stringify(excludeRegions, null, 2)}`); const errors: string[] = []; - const credentials = await sts.getCredentialsForAccountAndRole(accountId, assumeRoleName); + const credentials = await sts.getCredentialsForAccountAndRole(accountId.trim(), assumeRoleName.trim()); for (const region of regions) { console.log(`Deleting Default vpc in ${region}`); try { @@ -102,4 +102,4 @@ async function deleteVpc(ec2: EC2, vpcId: string, accountId: string, region: str errors.push(`${accountId}:${region}: ${error.code}: ${error.message}`); } return errors; -} +} \ No newline at end of file From 003a767d5c2f4f4724a8c48f315ae9e243df4d77 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Mon, 7 Sep 2020 21:21:53 +0530 Subject: [PATCH 07/20] Adding config to StoreOutputTask --- src/core/cdk/src/initial-setup.ts | 3 +++ src/core/cdk/src/tasks/store-outputs-task.ts | 6 ++++++ src/core/runtime/src/create-stack-set/create-stack-set.ts | 1 + src/core/runtime/src/delete-default-vpc/delete.ts | 2 +- 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 45ce17f37..ad3665dfc 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -579,6 +579,9 @@ export namespace InitialSetup { assumeRoleName: props.stateMachineExecutionRole, outputsTable: outputsTable.tableName, phaseNumber: phase, + configRepositoryName: props.configRepositoryName, + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', }, }), resultPath: 'DISCARD', diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 9313ae760..14211fa1f 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -41,6 +41,9 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { 'assumeRoleName.$': '$.assumeRoleName', 'outputsTable.$': '$.outputsTable', 'phaseNumber.$': '$.phaseNumber', + 'configRepositoryName.$': '$.configRepositoryName', + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', }, }); @@ -55,6 +58,9 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { 'assumeRoleName.$': '$.assumeRoleName', 'outputsTable.$': '$.outputsTable', 'phaseNumber.$': '$.phaseNumber', + 'configRepositoryName.$': '$.configRepositoryName', + 'configFilePath.$': '$.configFilePath', + 'configCommitId.$': '$.configCommitId', }, }); diff --git a/src/core/runtime/src/create-stack-set/create-stack-set.ts b/src/core/runtime/src/create-stack-set/create-stack-set.ts index 4e5b71bd5..a9ace286a 100644 --- a/src/core/runtime/src/create-stack-set/create-stack-set.ts +++ b/src/core/runtime/src/create-stack-set/create-stack-set.ts @@ -45,6 +45,7 @@ export const handler = async (input: CreateStackSetInput) => { OperationPreferences: { FailureTolerancePercentage: 100, MaxConcurrentPercentage: 100, + MaxConcurrentCount: 10, }, }); return { diff --git a/src/core/runtime/src/delete-default-vpc/delete.ts b/src/core/runtime/src/delete-default-vpc/delete.ts index b9e38206f..5ebcef4ee 100644 --- a/src/core/runtime/src/delete-default-vpc/delete.ts +++ b/src/core/runtime/src/delete-default-vpc/delete.ts @@ -102,4 +102,4 @@ async function deleteVpc(ec2: EC2, vpcId: string, accountId: string, region: str errors.push(`${accountId}:${region}: ${error.code}: ${error.message}`); } return errors; -} \ No newline at end of file +} From 8106be9671f7532fff10fb2f89259ed3ff889ff7 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 10:27:51 +0530 Subject: [PATCH 08/20] Adding get-account-info step in store-stack-outputs --- src/core/cdk/src/tasks/store-outputs-task.ts | 20 +++++-- src/core/runtime/src/get-account-info.ts | 54 +++++++++++++++++++ src/core/runtime/src/index.ts | 1 + .../runtime/src/store-stack-output-step.ts | 26 +++------ 4 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 src/core/runtime/src/get-account-info.ts diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 14211fa1f..75b7d3a3e 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -47,23 +47,33 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { }, }); + + const getAccountInfoResultPath = '$.account'; + const getAccountInfoTask = new CodeTask(scope, `Get Account Info`, { + resultPath: getAccountInfoResultPath, + functionPayload, + functionProps: { + role, + code: lambdaCode, + handler: 'index.getAccountInfo', + }, + }); + const storeAccountRegionOutputs = new sfn.Map(this, `Store Account Region Outputs`, { itemsPath: `$.regions`, resultPath: 'DISCARD', maxConcurrency: 10, parameters: { - 'accountId.$': '$.accountId', + 'account.$': '$.account', 'region.$': '$$.Map.Item.Value', 'acceleratorPrefix.$': '$.acceleratorPrefix', 'assumeRoleName.$': '$.assumeRoleName', 'outputsTable.$': '$.outputsTable', 'phaseNumber.$': '$.phaseNumber', - 'configRepositoryName.$': '$.configRepositoryName', - 'configFilePath.$': '$.configFilePath', - 'configCommitId.$': '$.configCommitId', }, }); + getAccountInfoTask.next(storeAccountRegionOutputs); const startTaskResultPath = '$.storeOutputsOutput'; const storeOutputsTask = new CodeTask(scope, `Store Outputs`, { resultPath: startTaskResultPath, @@ -76,7 +86,7 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { }); const pass = new sfn.Pass(this, 'Store Outputs Success'); - storeAccountOutputs.iterator(storeAccountRegionOutputs); + storeAccountOutputs.iterator(getAccountInfoTask); storeAccountRegionOutputs.iterator(storeOutputsTask); const chain = sfn.Chain.start(storeAccountOutputs).next(pass); diff --git a/src/core/runtime/src/get-account-info.ts b/src/core/runtime/src/get-account-info.ts new file mode 100644 index 000000000..82a291afc --- /dev/null +++ b/src/core/runtime/src/get-account-info.ts @@ -0,0 +1,54 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; +import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; +import { LoadConfigurationInput } from './load-configuration-step'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; + +export interface StoreStackOutputInput extends LoadConfigurationInput { + acceleratorPrefix: string; + assumeRoleName: string; + accountId: string; + region: string; + outputsTable: string; + phaseNumber: number; +} + +const organizations = new Organizations(); + +export const handler = async (input: StoreStackOutputInput) => { + console.log(`Retriving Account Info...`); + console.log(JSON.stringify(input, null, 2)); + + const { accountId, configCommitId, configFilePath, configRepositoryName } = input; + + // Retrieve Configuration from Code Commit with specific commitId + const acceleratorConfig = await loadAcceleratorConfig({ + repositoryName: configRepositoryName, + filePath: configFilePath, + commitId: configCommitId, + }); + + const awsAccount = await organizations.getAccount(accountId); + if (!awsAccount) { + throw new Error(`Unable retrive account from Organizations api for "${accountId}"`); + } + const configAccount = acceleratorConfig + .getAccountConfigs() + .find(([_, accountConfig]) => accountConfig.email === awsAccount.Email); + if (!configAccount) { + throw new Error(`Account didn't find in Configuration "${accountId}" with email ${awsAccount.Email}`); + } + const accountKey = configAccount?.[0]!; + const ou = configAccount?.[1].ou; + const ouPath = configAccount?.[1]['ou-path']; + const account: Account = { + arn: awsAccount.Arn!, + email: awsAccount.Email!, + id: accountId, + key: accountKey, + name: awsAccount.Name!, + ou: ou!, + ouPath, + }; + return account; +}; diff --git a/src/core/runtime/src/index.ts b/src/core/runtime/src/index.ts index d53445770..e51dbb036 100644 --- a/src/core/runtime/src/index.ts +++ b/src/core/runtime/src/index.ts @@ -23,6 +23,7 @@ export { handler as loadOrganizations } from './load-organizations-step'; export { handler as verifyFilesStep } from './verify-files-step'; export { handler as notifySMFailure } from './notify-statemachine-failure'; export { handler as notifySMSuccess } from './notify-statemachine-success'; +export { handler as getAccountInfo } from './get-account-info'; // TODO Replace with // export * as codebuild from './codebuild'; diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index 8907eb38c..fa3d526cc 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -5,11 +5,12 @@ import { StackOutput } from '@aws-accelerator/common-outputs/src/stack-output'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; +import { Account } from '@aws-accelerator/common-outputs/src/accounts'; -export interface StoreStackOutputInput extends LoadConfigurationInput { +export interface StoreStackOutputInput { acceleratorPrefix: string; assumeRoleName: string; - accountId: string; + account: Account; region: string; outputsTable: string; phaseNumber: number; @@ -17,7 +18,6 @@ export interface StoreStackOutputInput extends LoadConfigurationInput { const sts = new STS(); const dynamodb = new DynamoDB(); -const organizations = new Organizations(); export const handler = async (input: StoreStackOutputInput) => { console.log(`Storing stack output...`); @@ -26,31 +26,17 @@ export const handler = async (input: StoreStackOutputInput) => { const { acceleratorPrefix, assumeRoleName, - accountId, + account, region, outputsTable, phaseNumber, - configCommitId, - configFilePath, - configRepositoryName, } = input; - const credentials = await sts.getCredentialsForAccountAndRole(accountId, assumeRoleName); + const accountKey = account.key; + const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); const cfn = new CloudFormation(credentials, region); const stacks = cfn.listStacksGenerator({ StackStatusFilter: ['CREATE_COMPLETE', 'UPDATE_COMPLETE'], }); - // Retrieve Configuration from Code Commit with specific commitId - const acceleratorConfig = await loadAcceleratorConfig({ - repositoryName: configRepositoryName, - filePath: configFilePath, - commitId: configCommitId, - }); - - const awsAccount = await organizations.getAccount(accountId); - const configAccount = acceleratorConfig - .getAccountConfigs() - .find(([_, accountConfig]) => accountConfig.email === awsAccount?.Email); - const accountKey = configAccount?.[0]!; const outputs: StackOutput[] = []; for await (const summary of stacks) { From 9b41a058e23618b20f7976fa9158dc484823bed7 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 10:28:03 +0530 Subject: [PATCH 09/20] Prettier --- src/core/cdk/src/tasks/store-outputs-task.ts | 1 - src/core/runtime/src/store-stack-output-step.ts | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 75b7d3a3e..0d20f96a8 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -47,7 +47,6 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { }, }); - const getAccountInfoResultPath = '$.account'; const getAccountInfoTask = new CodeTask(scope, `Get Account Info`, { resultPath: getAccountInfoResultPath, diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index fa3d526cc..ee4c297dd 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -23,14 +23,7 @@ export const handler = async (input: StoreStackOutputInput) => { console.log(`Storing stack output...`); console.log(JSON.stringify(input, null, 2)); - const { - acceleratorPrefix, - assumeRoleName, - account, - region, - outputsTable, - phaseNumber, - } = input; + const { acceleratorPrefix, assumeRoleName, account, region, outputsTable, phaseNumber } = input; const accountKey = account.key; const credentials = await sts.getCredentialsForAccountAndRole(account.id, assumeRoleName); const cfn = new CloudFormation(credentials, region); From 77030a73c94ac10b87fbd80e2b8e8e04b000ce55 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 12:50:54 +0530 Subject: [PATCH 10/20] using updateItem instead of putItem --- .../cdk/src/tasks/run-across-accounts-task.ts | 2 +- .../runtime/src/store-stack-output-step.ts | 39 ++++++++++++++---- .../runtime/src/utils/dynamodb-requests.ts | 40 +++++++++++++++++++ 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/src/core/cdk/src/tasks/run-across-accounts-task.ts b/src/core/cdk/src/tasks/run-across-accounts-task.ts index fd9c5dc80..121c1ce11 100644 --- a/src/core/cdk/src/tasks/run-across-accounts-task.ts +++ b/src/core/cdk/src/tasks/run-across-accounts-task.ts @@ -66,7 +66,7 @@ export class RunAcrossAccountsTask extends sfn.StateMachineFragment { const mapTask = new sfn.Map(this, `${name} Map`, { itemsPath: '$.accounts', resultPath: '$.errors', - maxConcurrency: 10, + maxConcurrency: 50, parameters: { 'accountId.$': '$$.Map.Item.Value', assumeRoleName, diff --git a/src/core/runtime/src/store-stack-output-step.ts b/src/core/runtime/src/store-stack-output-step.ts index ee4c297dd..2a4b3ea1a 100644 --- a/src/core/runtime/src/store-stack-output-step.ts +++ b/src/core/runtime/src/store-stack-output-step.ts @@ -6,6 +6,7 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { LoadConfigurationInput } from './load-configuration-step'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; +import { getUpdateValueInput } from './utils/dynamodb-requests'; export interface StoreStackOutputInput { acceleratorPrefix: string; @@ -72,15 +73,39 @@ export const handler = async (input: StoreStackOutputInput) => { status: 'SUCCESS', }; } - await dynamodb.putItem({ - Item: { - id: { S: `${accountKey}-${region}-${phaseNumber}` }, - accountKey: { S: accountKey }, - region: { S: region }, - phase: { N: `${phaseNumber}` }, - outputValue: { S: JSON.stringify(outputs) }, + + const updateExpression = getUpdateValueInput([ + { + key: 'a', + name: 'accountKey', + type: 'S', + value: accountKey, + }, + { + key: 'r', + name: 'region', + type: 'S', + value: region, + }, + { + key: 'p', + name: 'phase', + type: 'N', + value: `${phaseNumber}`, }, + { + key: 'v', + name: 'outputValue', + type: 'S', + value: JSON.stringify(outputs), + }, + ]); + await dynamodb.updateItem({ TableName: outputsTable, + Key: { + id: { S: `${accountKey}-${region}-${phaseNumber}` }, + }, + ...updateExpression, }); return { status: 'SUCCESS', diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts index 1296ff033..b3f453a9a 100644 --- a/src/core/runtime/src/utils/dynamodb-requests.ts +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -31,3 +31,43 @@ export const getUpdateItemInput = ( }, }; }; + +interface Attribute { + key: string; + value: string; + name: string; + type: "S"|"N"|"B" ; +} + +/** + * + * @param attributes : Attribute[] + * key: needs to be unique + * + * returns updateExpression required for DynamoDB.updateItem + */ +export const getUpdateValueInput = ( + attributes: Attribute[], +) => { + if (attributes.length === 0) { + return; + } + const expAttributeNames: DynamoDB.ExpressionAttributeNameMap = {}; + const expAttributeValues: DynamoDB.ExpressionAttributeValueMap = {}; + let updateExtression: string = 'set '; + for (const att of attributes) { + const attributeValue: DynamoDB.AttributeValue = {}; + expAttributeNames[`#${att.key}`] = att.name; + attributeValue[att.type] = att.value; + expAttributeValues[`:${att.key}`] = attributeValue; + updateExtression += `#${att.key} = :${att.key},`; + } + // Remove "," if exisits in Last character + updateExtression = updateExtression.endsWith(',')? updateExtression.slice(0, -1): updateExtression; + + return { + ExpressionAttributeNames: expAttributeNames, + UpdateExpression: updateExtression, + ExpressionAttributeValues: expAttributeValues, + }; +}; From 4376201e94f85b0f0d4a60c8cfd64d3c457dc0d2 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 12:51:03 +0530 Subject: [PATCH 11/20] prettier --- src/core/runtime/src/utils/dynamodb-requests.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts index b3f453a9a..6ea3387b5 100644 --- a/src/core/runtime/src/utils/dynamodb-requests.ts +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -36,19 +36,17 @@ interface Attribute { key: string; value: string; name: string; - type: "S"|"N"|"B" ; + type: 'S' | 'N' | 'B'; } /** - * + * * @param attributes : Attribute[] * key: needs to be unique - * + * * returns updateExpression required for DynamoDB.updateItem */ -export const getUpdateValueInput = ( - attributes: Attribute[], -) => { +export const getUpdateValueInput = (attributes: Attribute[]) => { if (attributes.length === 0) { return; } @@ -63,7 +61,7 @@ export const getUpdateValueInput = ( updateExtression += `#${att.key} = :${att.key},`; } // Remove "," if exisits in Last character - updateExtression = updateExtression.endsWith(',')? updateExtression.slice(0, -1): updateExtression; + updateExtression = updateExtression.endsWith(',') ? updateExtression.slice(0, -1) : updateExtression; return { ExpressionAttributeNames: expAttributeNames, From b56de11a41cd0239ca9362f5436f97ac08be5568 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 12:57:16 +0530 Subject: [PATCH 12/20] Lint fix --- src/core/runtime/src/get-account-info.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/src/get-account-info.ts b/src/core/runtime/src/get-account-info.ts index 82a291afc..20c2c5c8a 100644 --- a/src/core/runtime/src/get-account-info.ts +++ b/src/core/runtime/src/get-account-info.ts @@ -38,7 +38,7 @@ export const handler = async (input: StoreStackOutputInput) => { if (!configAccount) { throw new Error(`Account didn't find in Configuration "${accountId}" with email ${awsAccount.Email}`); } - const accountKey = configAccount?.[0]!; + const accountKey = configAccount?.[0]; const ou = configAccount?.[1].ou; const ouPath = configAccount?.[1]['ou-path']; const account: Account = { @@ -47,7 +47,7 @@ export const handler = async (input: StoreStackOutputInput) => { id: accountId, key: accountKey, name: awsAccount.Name!, - ou: ou!, + ou, ouPath, }; return account; From 03a161db6428c3ada151197127f65cf8d93a7095 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 13:44:23 +0530 Subject: [PATCH 13/20] Removing ShortAccount --- src/core/runtime/src/delete-default-vpc/delete.ts | 2 +- src/lib/common-outputs/src/accounts.ts | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/core/runtime/src/delete-default-vpc/delete.ts b/src/core/runtime/src/delete-default-vpc/delete.ts index 5ebcef4ee..fc914076c 100644 --- a/src/core/runtime/src/delete-default-vpc/delete.ts +++ b/src/core/runtime/src/delete-default-vpc/delete.ts @@ -38,7 +38,7 @@ export const handler = async (input: DeleteVPCInput): Promise => { console.log(`${accountId}: Excluding Deletion of Default VPC for regions from account "${accountId}"...`); console.log(`${accountId}: ${JSON.stringify(excludeRegions, null, 2)}`); const errors: string[] = []; - const credentials = await sts.getCredentialsForAccountAndRole(accountId.trim(), assumeRoleName.trim()); + const credentials = await sts.getCredentialsForAccountAndRole(accountId, assumeRoleName); for (const region of regions) { console.log(`Deleting Default vpc in ${region}`); try { diff --git a/src/lib/common-outputs/src/accounts.ts b/src/lib/common-outputs/src/accounts.ts index f818139a0..664e52c3b 100644 --- a/src/lib/common-outputs/src/accounts.ts +++ b/src/lib/common-outputs/src/accounts.ts @@ -13,12 +13,7 @@ export interface Account { ouPath?: string; } -export interface ShortAccount { - key: string; - id: string; -} - -export function getAccountId(accounts: Account[] | ShortAccount[], accountKey: string): string | undefined { +export function getAccountId(accounts: Account[], accountKey: string): string | undefined { const account = accounts.find(a => a.key === accountKey); if (!account) { console.warn(`Cannot find account with key "${accountKey}"`); From a9294c02f4b0c88f48550857d5de6d31f07013c3 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 14:03:49 +0530 Subject: [PATCH 14/20] Adding Pagination to scan --- src/core/runtime/src/utils/load-outputs.ts | 4 ++-- src/deployments/cdk/src/utils/outputs.ts | 4 ++-- src/lib/common/src/aws/dynamodb.ts | 13 +++++++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/core/runtime/src/utils/load-outputs.ts b/src/core/runtime/src/utils/load-outputs.ts index 83850d0ae..d0745ce2f 100644 --- a/src/core/runtime/src/utils/load-outputs.ts +++ b/src/core/runtime/src/utils/load-outputs.ts @@ -6,11 +6,11 @@ export async function loadOutputs(tableName: string, client: DynamoDB): Promise< const outputsResponse = await client.scan({ TableName: tableName, }); - if (!outputsResponse.Items) { + if (!outputsResponse) { console.warn(`Did not find outputs in DynamoDB table "${tableName}"`); return []; } - for (const item of outputsResponse.Items) { + for (const item of outputsResponse) { const cVal = JSON.parse(item.outputValue.S!); outputs.push(...cVal); } diff --git a/src/deployments/cdk/src/utils/outputs.ts b/src/deployments/cdk/src/utils/outputs.ts index ee4739948..b87033c13 100644 --- a/src/deployments/cdk/src/utils/outputs.ts +++ b/src/deployments/cdk/src/utils/outputs.ts @@ -25,11 +25,11 @@ export async function loadStackOutputs(): Promise { const outputsResponse = await dynamodb.scan({ TableName: outputTableName, }); - if (!outputsResponse.Items) { + if (!outputsResponse) { console.warn(`Did not find outputs in DynamoDB table "${outputTableName}"`); return []; } - for (const item of outputsResponse.Items) { + for (const item of outputsResponse) { const cVal = JSON.parse(item.outputValue.S!); outputs.push(...cVal); } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 9b582181a..85e56a631 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -19,8 +19,17 @@ export class DynamoDB { await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); } - async scan(props: dynamodb.ScanInput): Promise { - return throttlingBackOff(() => this.client.scan(props).promise()); + async scan(props: dynamodb.ScanInput): Promise { + const items: dynamodb.ItemList = []; + let token: dynamodb.Key | undefined; + // TODO: Use common listgenerator when this api supports nextToken + do { + const response = await throttlingBackOff(() => this.client.scan(props).promise()) + token = response.LastEvaluatedKey; + props.ExclusiveStartKey = token; + items.push(...response.Items!); + } while (token); + return items; } async putItem(props: dynamodb.PutItemInput): Promise { From 31e206ee3247bd728d87cf4cd65419547804bb5a Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 14:03:58 +0530 Subject: [PATCH 15/20] prettier --- src/lib/common/src/aws/dynamodb.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 85e56a631..0a36a170a 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -24,7 +24,7 @@ export class DynamoDB { let token: dynamodb.Key | undefined; // TODO: Use common listgenerator when this api supports nextToken do { - const response = await throttlingBackOff(() => this.client.scan(props).promise()) + const response = await throttlingBackOff(() => this.client.scan(props).promise()); token = response.LastEvaluatedKey; props.ExclusiveStartKey = token; items.push(...response.Items!); From 7aa8f9cc985e9948613c26f5f5922e7a7fb88e29 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 14:52:42 +0530 Subject: [PATCH 16/20] Fixing unittestes with respect to return in load configuration --- .../test/load-configuration-step.spec.ts | 53 ++----------------- 1 file changed, 3 insertions(+), 50 deletions(-) diff --git a/src/core/runtime/test/load-configuration-step.spec.ts b/src/core/runtime/test/load-configuration-step.spec.ts index 321471d0c..ebbce22fd 100644 --- a/src/core/runtime/test/load-configuration-step.spec.ts +++ b/src/core/runtime/test/load-configuration-step.spec.ts @@ -16,55 +16,7 @@ test('the handler should be successfully return when the configuration is correc configRepositoryName: 'PBMMAccel-Repo-Config', configCommitId: 'fasdjfkhsdf', }); - - expect(result.accounts).toHaveLength(6); - - expect(result.accounts).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - accountKey: 'primary-key', - accountName: 'primary', - emailAddress: 'lz@amazon.com', - landingZoneAccountType: 'primary', - organizationalUnit: 'core', - }), - expect.objectContaining({ - accountKey: 'log-archive-key', - accountName: 'log-archive', - emailAddress: 'lz+log-archive@amazon.com', - landingZoneAccountType: 'log-archive', - organizationalUnit: 'core', - }), - expect.objectContaining({ - accountKey: 'security-key', - accountName: 'security', - emailAddress: 'lz+security@amazon.com', - landingZoneAccountType: 'security', - organizationalUnit: 'core', - }), - expect.objectContaining({ - accountKey: 'shared-services-key', - accountName: 'shared-services', - emailAddress: 'lz+shared-services@amazon.com', - landingZoneAccountType: 'shared-services', - organizationalUnit: 'core', - }), - expect.objectContaining({ - accountKey: 'shared-network-key', - accountName: 'shared-network', - emailAddress: 'lz+shared-network@amazon.com', - landingZoneAccountType: undefined, - organizationalUnit: 'core', - }), - expect.objectContaining({ - accountKey: 'operations-key', - accountName: 'operations', - emailAddress: 'lz+operations@amazon.com', - landingZoneAccountType: undefined, - organizationalUnit: 'core', - }), - ]), - ); + expect(result); }); test('the handler should be successfully return when a mandatory account is missing', async () => { @@ -79,7 +31,8 @@ test('the handler should be successfully return when a mandatory account is miss configCommitId: '', }); - expect(result.accounts).toHaveLength(6); + // Returns only Accounts that needs to be created + expect(result.accounts).toHaveLength(1); }); test('the handler should throw an error when the Accelerator config name does not match the account name', async () => { From 4e448af12a8a3e45ccdde160ac3145fa5e7c2119 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 15:31:02 +0530 Subject: [PATCH 17/20] Review fixes --- src/core/cdk/src/tasks/store-outputs-task.ts | 6 ++---- src/core/runtime/src/add-scp-step.ts | 7 ++----- .../src/configuration/load-landing-zone-config.ts | 1 + .../src/configuration/load-organizations-config.ts | 2 +- src/core/runtime/src/get-account-info.ts | 2 +- src/core/runtime/src/utils/dynamodb-requests.ts | 10 +++++----- src/core/runtime/src/utils/load-organizations.ts | 14 ++++++++++++++ 7 files changed, 26 insertions(+), 16 deletions(-) create mode 100644 src/core/runtime/src/utils/load-organizations.ts diff --git a/src/core/cdk/src/tasks/store-outputs-task.ts b/src/core/cdk/src/tasks/store-outputs-task.ts index 0d20f96a8..f94932bb8 100644 --- a/src/core/cdk/src/tasks/store-outputs-task.ts +++ b/src/core/cdk/src/tasks/store-outputs-task.ts @@ -47,9 +47,8 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { }, }); - const getAccountInfoResultPath = '$.account'; const getAccountInfoTask = new CodeTask(scope, `Get Account Info`, { - resultPath: getAccountInfoResultPath, + resultPath: '$.account', functionPayload, functionProps: { role, @@ -73,9 +72,8 @@ export class StoreOutputsTask extends sfn.StateMachineFragment { }); getAccountInfoTask.next(storeAccountRegionOutputs); - const startTaskResultPath = '$.storeOutputsOutput'; const storeOutputsTask = new CodeTask(scope, `Store Outputs`, { - resultPath: startTaskResultPath, + resultPath: '$.storeOutputsOutput', functionPayload, functionProps: { role, diff --git a/src/core/runtime/src/add-scp-step.ts b/src/core/runtime/src/add-scp-step.ts index 27517b646..a31d38d4e 100644 --- a/src/core/runtime/src/add-scp-step.ts +++ b/src/core/runtime/src/add-scp-step.ts @@ -7,6 +7,7 @@ import { ServiceControlPolicy } from '@aws-accelerator/common/src/scp'; import { loadOutputs } from './utils/load-outputs'; import { getItemInput } from './utils/dynamodb-requests'; import { loadAccounts } from './utils/load-accounts'; +import { loadOrganizations } from './utils/load-organizations'; interface AddScpInput extends LoadConfigurationInput { acceleratorPrefix: string; @@ -40,6 +41,7 @@ export const handler = async (input: AddScpInput) => { const outputs = await loadOutputs(outputTableName, dynamodb); const accounts = await loadAccounts(parametersTableName, dynamodb); + const organizationalUnits = await loadOrganizations(parametersTableName, dynamodb); // Find the SCP artifact output const artifactOutput = ArtifactOutputFinder.findOneByName({ @@ -68,11 +70,6 @@ export const handler = async (input: AddScpInput) => { // Find roots to attach FullAWSAccess const rootIds = await scps.organizationRoots(); - const organizationsResponse = await dynamodb.getItem(getItemInput(parametersTableName, 'organizations')); - if (!organizationsResponse.Item) { - throw new Error(`No organizations found in DynamoDB "${parametersTableName}"`); - } - const organizationalUnits: OrganizationalUnit[] = JSON.parse(organizationsResponse.Item.value.S!); // Find Accelerator accounts and OUs to attach FullAWSAccess const acceleratorOuIds = organizationalUnits.map(ou => ou.ouId); const acceleratorAccountIds = accounts.map(a => a.id); diff --git a/src/core/runtime/src/configuration/load-landing-zone-config.ts b/src/core/runtime/src/configuration/load-landing-zone-config.ts index 1f6467556..5735d6431 100644 --- a/src/core/runtime/src/configuration/load-landing-zone-config.ts +++ b/src/core/runtime/src/configuration/load-landing-zone-config.ts @@ -306,6 +306,7 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), regions: config['global-options']['supported-regions'], warnings, diff --git a/src/core/runtime/src/configuration/load-organizations-config.ts b/src/core/runtime/src/configuration/load-organizations-config.ts index 2ef54c20f..e2f542cb1 100644 --- a/src/core/runtime/src/configuration/load-organizations-config.ts +++ b/src/core/runtime/src/configuration/load-organizations-config.ts @@ -228,7 +228,7 @@ export const handler = async (input: LoadConfigurationInput): Promise !acc.accountId), regions: config['global-options']['supported-regions'], warnings, diff --git a/src/core/runtime/src/get-account-info.ts b/src/core/runtime/src/get-account-info.ts index 20c2c5c8a..186b1eaae 100644 --- a/src/core/runtime/src/get-account-info.ts +++ b/src/core/runtime/src/get-account-info.ts @@ -16,7 +16,7 @@ export interface StoreStackOutputInput extends LoadConfigurationInput { const organizations = new Organizations(); export const handler = async (input: StoreStackOutputInput) => { - console.log(`Retriving Account Info...`); + console.log(`Get Account Info...`); console.log(JSON.stringify(input, null, 2)); const { accountId, configCommitId, configFilePath, configRepositoryName } = input; diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts index 6ea3387b5..6c8691f50 100644 --- a/src/core/runtime/src/utils/dynamodb-requests.ts +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -52,20 +52,20 @@ export const getUpdateValueInput = (attributes: Attribute[]) => { } const expAttributeNames: DynamoDB.ExpressionAttributeNameMap = {}; const expAttributeValues: DynamoDB.ExpressionAttributeValueMap = {}; - let updateExtression: string = 'set '; + let updateExpression: string = 'set '; for (const att of attributes) { const attributeValue: DynamoDB.AttributeValue = {}; expAttributeNames[`#${att.key}`] = att.name; attributeValue[att.type] = att.value; expAttributeValues[`:${att.key}`] = attributeValue; - updateExtression += `#${att.key} = :${att.key},`; + updateExpression += `#${att.key} = :${att.key},`; } - // Remove "," if exisits in Last character - updateExtression = updateExtression.endsWith(',') ? updateExtression.slice(0, -1) : updateExtression; + // Remove "," if exists as last character + updateExpression = updateExpression.endsWith(',') ? updateExpression.slice(0, -1) : updateExpression; return { ExpressionAttributeNames: expAttributeNames, - UpdateExpression: updateExtression, + UpdateExpression: updateExpression, ExpressionAttributeValues: expAttributeValues, }; }; diff --git a/src/core/runtime/src/utils/load-organizations.ts b/src/core/runtime/src/utils/load-organizations.ts new file mode 100644 index 000000000..1a8137ee3 --- /dev/null +++ b/src/core/runtime/src/utils/load-organizations.ts @@ -0,0 +1,14 @@ +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; + +export async function loadOrganizations(tableName: string, client: DynamoDB): Promise { + const itemInput = { + TableName: tableName, + Key: { id: { S: `organizations` } }, + }; + const organizationsResponse = await client.getItem(itemInput); + if (!organizationsResponse.Item) { + throw new Error(`No organizations found in DynamoDB "${tableName}"`); + } + return JSON.parse(organizationsResponse.Item.value.S!); +} From 2106a35418015600d88b4ac0e02a903d3e29fd85 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 19:20:25 +0530 Subject: [PATCH 18/20] Initial Push --- .../src/add-tags-to-shared-resources-step.ts | 8 ++++++- src/deployments/cdk/src/apps/phase-2.ts | 1 + .../common/add-tags-to-resources-output.ts | 1 + .../cdk/src/common/vpc-subnet-sharing.ts | 2 ++ src/deployments/cdk/src/common/vpc.ts | 1 + .../deployments/firewall/cluster/step-2.ts | 1 + .../test/apps/unsupported-changes.mocks.ts | 24 +++++++++++++++++++ src/lib/common/src/aws/resource-tagging.ts | 3 ++- 8 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/core/runtime/src/add-tags-to-shared-resources-step.ts b/src/core/runtime/src/add-tags-to-shared-resources-step.ts index 5473b8889..60589dffc 100644 --- a/src/core/runtime/src/add-tags-to-shared-resources-step.ts +++ b/src/core/runtime/src/add-tags-to-shared-resources-step.ts @@ -21,6 +21,7 @@ interface AddTagToResourceOutput { resourceType: string; targetAccountIds: string[]; tags: Tag[]; + region: string; } type AddTagToResourceOutputs = AddTagToResourceOutput[]; @@ -48,7 +49,7 @@ export const handler = async (input: CreateTagsRequestInput) => { const credentials = await sts.getCredentialsForAccountAndRole(targetAccountId, assumeRoleName); if (ALLOWED_RESOURCE_TYPES.includes(resourceType)) { try { - const tagResources = new TagResources(credentials); + const tagResources = new TagResources(credentials, addTagsToResources.region); await tagResources.createTags({ Resources: [resourceId], Tags: tags.map(t => ({ Key: t.key, Value: t.value })), @@ -68,3 +69,8 @@ export const handler = async (input: CreateTagsRequestInput) => { statusReason: `Added tags for all the shared resources`, }; }; + +handler({ + "assumeRoleName": "PBMMAccel-PipelineRole", + "outputTableName": "PBMMAccel-Outputs", +}); diff --git a/src/deployments/cdk/src/apps/phase-2.ts b/src/deployments/cdk/src/apps/phase-2.ts index 0e8b67f52..3d0ac4ab8 100644 --- a/src/deployments/cdk/src/apps/phase-2.ts +++ b/src/deployments/cdk/src/apps/phase-2.ts @@ -200,6 +200,7 @@ export async function deploy({ acceleratorConfig, accountStacks, accounts, conte resourceType: 'security-group', targetAccountIds: [accountId], tags: securityGroup.tags.renderTags(), + region: securityGroupStack.region, })), }); } diff --git a/src/deployments/cdk/src/common/add-tags-to-resources-output.ts b/src/deployments/cdk/src/common/add-tags-to-resources-output.ts index b0b79c4be..3db0c8616 100644 --- a/src/deployments/cdk/src/common/add-tags-to-resources-output.ts +++ b/src/deployments/cdk/src/common/add-tags-to-resources-output.ts @@ -11,6 +11,7 @@ export interface AddTagsToResource { resourceType: 'subnet' | 'security-group' | 'vpc' | 'tgw-attachment'; targetAccountIds: string[]; tags: Tag[]; + region: string; } export interface AddTagsToResourcesOutputProps extends Omit { diff --git a/src/deployments/cdk/src/common/vpc-subnet-sharing.ts b/src/deployments/cdk/src/common/vpc-subnet-sharing.ts index 484c254c6..1126bf5c6 100644 --- a/src/deployments/cdk/src/common/vpc-subnet-sharing.ts +++ b/src/deployments/cdk/src/common/vpc-subnet-sharing.ts @@ -125,6 +125,7 @@ export class VpcSubnetSharing extends cdk.Construct { sourceAccountId: o.sourceAccountId, targetAccountIds: o.targetAccountIds, tags: o.subnet.tags.renderTags(), + region: cdk.Aws.REGION, })), }); @@ -137,6 +138,7 @@ export class VpcSubnetSharing extends cdk.Construct { sourceAccountId: o.sourceAccountId, targetAccountIds: o.targetAccountIds, tags: vpc.tags.renderTags(), + region: cdk.Aws.REGION, })), }); } diff --git a/src/deployments/cdk/src/common/vpc.ts b/src/deployments/cdk/src/common/vpc.ts index c6943f171..174a19267 100644 --- a/src/deployments/cdk/src/common/vpc.ts +++ b/src/deployments/cdk/src/common/vpc.ts @@ -346,6 +346,7 @@ export class Vpc extends cdk.Construct implements constructs.Vpc { resourceType: 'tgw-attachment', tags: tgwAttachment!.resource.tags.renderTags(), targetAccountIds: [ownerAccountId], + region: cdk.Aws.REGION, }, ], }); diff --git a/src/deployments/cdk/src/deployments/firewall/cluster/step-2.ts b/src/deployments/cdk/src/deployments/firewall/cluster/step-2.ts index 81b0481d2..c1a50be7d 100644 --- a/src/deployments/cdk/src/deployments/firewall/cluster/step-2.ts +++ b/src/deployments/cdk/src/deployments/firewall/cluster/step-2.ts @@ -174,6 +174,7 @@ async function createCustomerGateways(props: { value: `${prefix}_att`, }, ], + region: cdk.Aws.REGION, }); const associateConfig = attachConfig['tgw-rt-associate'] || []; diff --git a/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts b/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts index d05aabc7e..959a1640a 100644 --- a/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts +++ b/src/deployments/cdk/test/apps/unsupported-changes.mocks.ts @@ -324,6 +324,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Web_Central_aza_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-075ca0d93563a4d6e', @@ -334,6 +335,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Web_Central_azb_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-0b70e3e849949cc0c', @@ -344,6 +346,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'App_Central_aza_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-04e252c39434bbb19', @@ -354,6 +357,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'App_Central_azb_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-00bbd856fec06c15b', @@ -364,6 +368,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Data_Central_aza_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-0ace127f373a1b274', @@ -374,6 +379,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Data_Central_azb_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-05eff632d6af56d17', @@ -384,6 +390,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Mgmt_Central_aza_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-03dc090bc09c84439', @@ -394,6 +401,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Mgmt_Central_azb_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-02809ab1988c1ec82', @@ -404,6 +412,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'GCWide_Central_aza_net' }, ], + region: 'ca-central-1', }, { resourceId: 'subnet-0f4514137baa07759', @@ -414,6 +423,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'GCWide_Central_azb_net' }, ], + region: 'ca-central-1', }, ], }), @@ -434,6 +444,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -444,6 +455,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -454,6 +466,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -464,6 +477,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -474,6 +488,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -484,6 +499,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -494,6 +510,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -504,6 +521,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -514,6 +532,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, { resourceId: 'vpc-0d0b4cd029857165a', @@ -524,6 +543,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Central_vpc' }, ], + region: 'ca-central-1', }, ], }), @@ -851,6 +871,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Mgmt_sg' }, ], + region: 'ca-central-1', }, { resourceId: 'sg-0c91bf1bc94e5797f', @@ -860,6 +881,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Web_sg' }, ], + region: 'ca-central-1', }, { resourceId: 'sg-08f5b682020bf1c61', @@ -869,6 +891,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'App_sg' }, ], + region: 'ca-central-1', }, { resourceId: 'sg-041abd65f62750b6e', @@ -878,6 +901,7 @@ export function createPhaseInput(): Omit { { key: 'Accelerator', value: 'PBMM' }, { key: 'Name', value: 'Data_sg' }, ], + region: 'ca-central-1', }, ], }), diff --git a/src/lib/common/src/aws/resource-tagging.ts b/src/lib/common/src/aws/resource-tagging.ts index 4c8fd7318..be12594af 100644 --- a/src/lib/common/src/aws/resource-tagging.ts +++ b/src/lib/common/src/aws/resource-tagging.ts @@ -5,9 +5,10 @@ import { throttlingBackOff } from './backoff'; export class TagResources { private readonly client: aws.EC2; - public constructor(credentials?: aws.Credentials) { + public constructor(credentials?: aws.Credentials, region?: string) { this.client = new aws.EC2({ credentials, + region, }); } From 92e9e2976235de4389be53f4df19976c6960bc14 Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Tue, 8 Sep 2020 19:20:41 +0530 Subject: [PATCH 19/20] Prettier --- src/core/runtime/src/add-tags-to-shared-resources-step.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/core/runtime/src/add-tags-to-shared-resources-step.ts b/src/core/runtime/src/add-tags-to-shared-resources-step.ts index 60589dffc..a6d076737 100644 --- a/src/core/runtime/src/add-tags-to-shared-resources-step.ts +++ b/src/core/runtime/src/add-tags-to-shared-resources-step.ts @@ -70,7 +70,7 @@ export const handler = async (input: CreateTagsRequestInput) => { }; }; -handler({ - "assumeRoleName": "PBMMAccel-PipelineRole", - "outputTableName": "PBMMAccel-Outputs", -}); +// handler({ +// assumeRoleName: 'PBMMAccel-PipelineRole', +// outputTableName: 'PBMMAccel-Outputs', +// }); From 0315c9e086ace6b11aaa07d6089aba0d237999fa Mon Sep 17 00:00:00 2001 From: Naveen Kumar Date: Wed, 9 Sep 2020 10:45:05 +0530 Subject: [PATCH 20/20] removing comments --- src/core/runtime/src/add-tags-to-shared-resources-step.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/core/runtime/src/add-tags-to-shared-resources-step.ts b/src/core/runtime/src/add-tags-to-shared-resources-step.ts index a6d076737..51164db50 100644 --- a/src/core/runtime/src/add-tags-to-shared-resources-step.ts +++ b/src/core/runtime/src/add-tags-to-shared-resources-step.ts @@ -69,8 +69,3 @@ export const handler = async (input: CreateTagsRequestInput) => { statusReason: `Added tags for all the shared resources`, }; }; - -// handler({ -// assumeRoleName: 'PBMMAccel-PipelineRole', -// outputTableName: 'PBMMAccel-Outputs', -// });