From 866e50eeddb8a8cd24a3efbdcadb63705ded9b70 Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 1 Sep 2020 17:40:40 +0530 Subject: [PATCH 1/8] moved configs from secrets manager to dynamodb --- src/core/cdk/package.json | 7 +-- src/core/cdk/src/initial-setup.ts | 41 +++++++--------- src/core/runtime/src/load-accounts-step.ts | 26 ++++++---- src/core/runtime/src/load-limits-step.ts | 25 ++++++---- .../runtime/src/load-organizations-step.ts | 18 +++---- src/core/runtime/src/ou-validation.ts | 48 ++++++++++++------- src/deployments/cdk/src/utils/accounts.ts | 33 +++++++++---- src/deployments/cdk/src/utils/limits.ts | 23 +++++---- .../cdk/src/utils/organizations.ts | 23 +++++---- src/lib/common/src/aws/dynamodb.ts | 45 +++++++++++++++++ 10 files changed, 191 insertions(+), 98 deletions(-) create mode 100644 src/lib/common/src/aws/dynamodb.ts diff --git a/src/core/cdk/package.json b/src/core/cdk/package.json index 4a35cb3c4..3d28984c6 100644 --- a/src/core/cdk/package.json +++ b/src/core/cdk/package.json @@ -23,10 +23,13 @@ "typescript": "3.8.3" }, "dependencies": { + "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", + "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", "@aws-cdk/aws-cloudformation": "1.46.0", "@aws-cdk/aws-codebuild": "1.46.0", "@aws-cdk/aws-codepipeline": "1.46.0", "@aws-cdk/aws-codepipeline-actions": "1.46.0", + "@aws-cdk/aws-dynamodb": "1.46.0", "@aws-cdk/aws-events": "1.46.0", "@aws-cdk/aws-events-targets": "1.46.0", "@aws-cdk/aws-iam": "1.46.0", @@ -36,13 +39,11 @@ "@aws-cdk/aws-s3": "1.46.0", "@aws-cdk/aws-s3-assets": "1.46.0", "@aws-cdk/aws-s3-deployment": "1.46.0", + "@aws-cdk/aws-secretsmanager": "1.46.0", "@aws-cdk/aws-sns": "1.46.0", "@aws-cdk/aws-stepfunctions": "1.46.0", "@aws-cdk/aws-stepfunctions-tasks": "1.46.0", - "@aws-cdk/aws-secretsmanager": "1.46.0", "@aws-cdk/core": "1.46.0", - "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", - "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", "@types/cfn-response": "^1.0.3", "aws-sdk": "2.668.0", "cfn-response": "^1.0.1", diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 04ed554c6..25ab61fdc 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -19,6 +19,7 @@ import { CreateStackTask } from './tasks/create-stack-task'; import { RunAcrossAccountsTask } from './tasks/run-across-accounts-task'; import * as fs from 'fs'; import * as sns from '@aws-cdk/aws-sns'; +import * as dynamodb from '@aws-cdk/aws-dynamodb'; export namespace InitialSetup { export interface CommonProps { @@ -67,22 +68,10 @@ export namespace InitialSetup { const stack = cdk.Stack.of(this); - const accountsSecret = new secrets.Secret(this, 'Accounts', { - secretName: 'accelerator/accounts', - description: 'This secret contains the information about the accounts that are used for deployment.', + const parametersTable = new dynamodb.Table(this, 'ParametersTable', { + partitionKey: { name: 'id', type: dynamodb.AttributeType.STRING }, + tableName: `${props.acceleratorPrefix}Parameters`, }); - setSecretValue(accountsSecret, '[]'); - - const limitsSecret = new secrets.Secret(this, 'Limits', { - secretName: 'accelerator/limits', - description: 'This secret contains a copy of the service limits of the Accelerator accounts.', - }); - - const organizationsSecret = new secrets.Secret(this, 'Organizations', { - secretName: 'accelerator/organizations', - description: 'This secret contains the information about the organizations that are used for deployment.', - }); - setSecretValue(organizationsSecret, '[]'); // This is the maximum time before a build times out // The role used by the build should allow this session duration @@ -96,6 +85,7 @@ export namespace InitialSetup { new iam.ServicePrincipal('codebuild.amazonaws.com'), new iam.ServicePrincipal('lambda.amazonaws.com'), new iam.ServicePrincipal('events.amazonaws.com'), + new iam.ServicePrincipal('dynamodb.amazonaws.com'), ), managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('AdministratorAccess')], maxSessionDuration: buildTimeout, @@ -121,9 +111,10 @@ export namespace InitialSetup { ACCELERATOR_EXECUTION_ROLE_NAME: props.stateMachineExecutionRole, CDK_PLUGIN_ASSUME_ROLE_NAME: props.stateMachineExecutionRole, CDK_PLUGIN_ASSUME_ROLE_DURATION: `${buildTimeout.toSeconds()}`, - ACCOUNTS_SECRET_ID: accountsSecret.secretArn, - LIMITS_SECRET_ID: limitsSecret.secretArn, - ORGANIZATIONS_SECRET_ID: organizationsSecret.secretArn, + ACCOUNTS_ITEM_ID: 'accounts', + LIMITS_ITEM_ID: 'limits', + ORGANIZATIONS_ITEM_ID: 'organizations', + DYNAMODB_PARAMETERS_TABLE_NAME: parametersTable.tableName, }, }); @@ -300,7 +291,8 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - organizationsSecretId: organizationsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'organizations', configRepositoryName: props.configRepositoryName, 'configFilePath.$': '$.configuration.configFilePath', 'configCommitId.$': '$.configuration.configCommitId', @@ -315,7 +307,8 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - accountsSecretId: accountsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'accounts', 'configuration.$': '$.configuration', }, resultPath: '$', @@ -430,7 +423,8 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - limitsSecretId: limitsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'limits', assumeRoleName: props.stateMachineExecutionRole, 'accounts.$': '$.accounts', }, @@ -448,8 +442,9 @@ export namespace InitialSetup { 'configFilePath.$': '$.configuration.configFilePath', 'configCommitId.$': '$.configuration.configCommitId', acceleratorPrefix: props.acceleratorPrefix, - accountsSecretId: accountsSecret.secretArn, - organizationsSecretId: organizationsSecret.secretArn, + parametersTableName: parametersTable.tableName, + organizationsItemId: 'organizations', + accountsItemId: 'accounts', configBranch: props.configBranchName, 'configRootFilePath.$': '$.configuration.configRootFilePath', }, diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index 935ef61b9..7549157fe 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,11 +1,12 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export interface LoadAccountsInput { - accountsSecretId: string; + parametersTableName: string; + itemId: string; configuration: LoadConfigurationOutput; } @@ -15,11 +16,13 @@ export interface LoadAccountsOutput { regions: string[]; } +const dynamoDB = new DynamoDB(); + export const handler = async (input: LoadAccountsInput): Promise => { console.log(`Loading accounts...`); console.log(JSON.stringify(input, null, 2)); - const { accountsSecretId, configuration } = input; + const { parametersTableName, configuration, itemId } = input; // The first step is to load all the execution roles const organizations = new Organizations(); @@ -27,6 +30,10 @@ export const handler = async (input: LoadAccountsInput): Promise account.Status === 'ACTIVE'); const accounts = []; + + const chunk = (accounts: Account[], size: number) => + Array.from({ length: Math.ceil(accounts.length / size) }, (v, i) => accounts.slice(i * size, i * size + size)); + for (const accountConfig of configuration.accounts) { let organizationAccount; organizationAccount = activeAccounts.find(a => { @@ -68,12 +75,13 @@ export const handler = async (input: LoadAccountsInput): Promise { console.log(`Loading limits...`); console.log(JSON.stringify(input, null, 2)); - const { configRepositoryName, configFilePath, limitsSecretId, accounts, assumeRoleName, configCommitId } = input; + const { + configRepositoryName, + configFilePath, + parametersTableName, + accounts, + assumeRoleName, + configCommitId, + itemId, + } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -147,10 +158,6 @@ export const handler = async (input: LoadLimitsInput) => { } } - // Store the limits in the secrets manager - const secrets = new SecretsManager(); - await secrets.putSecretValue({ - SecretId: limitsSecretId, - SecretString: JSON.stringify(limits, null, 2), - }); + // Store the limits in the dynamodb + await dynamoDB.putItem(parametersTableName, itemId, JSON.stringify(limits, null, 2)); }; diff --git a/src/core/runtime/src/load-organizations-step.ts b/src/core/runtime/src/load-organizations-step.ts index a8c0beee0..d5fafc386 100644 --- a/src/core/runtime/src/load-organizations-step.ts +++ b/src/core/runtime/src/load-organizations-step.ts @@ -1,26 +1,27 @@ import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export interface LoadOrganizationsInput extends LoadConfigurationInput { - organizationsSecretId: string; + parametersTableName: string; + itemId: string; } export type LoadOrganizationsOutput = { organizationalUnits: OrganizationalUnit[]; }; -const secrets = new SecretsManager(); const organizations = new Organizations(); +const dynamoDB = new DynamoDB(); export const handler = async (input: LoadOrganizationsInput): Promise => { console.log('Load Organizations ...'); console.log(JSON.stringify(input, null, 2)); const organizationalUnits: OrganizationalUnit[] = []; - const { organizationsSecretId, configCommitId, configFilePath, configRepositoryName } = input; + const { configCommitId, configFilePath, configRepositoryName, parametersTableName, itemId } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ repositoryName: configRepositoryName, @@ -44,11 +45,10 @@ export const handler = async (input: LoadOrganizationsInput): Promise => { configRepositoryName, configCommitId, acceleratorPrefix, - accountsSecretId, - organizationsSecretId, + parametersTableName, + organizationsItemId, + accountsItemId, configBranch, configRootFilePath, } = input; @@ -60,8 +62,8 @@ export const handler = async (input: ValdationInput): Promise => { let rootConfig = getFormattedObject(rootConfigString, format); let config = previousConfig; - const previousAccounts = await loadAccounts(accountsSecretId); - const previousOrganizationalUnits = await loadOrganizations(organizationsSecretId); + const previousAccounts = await loadAccounts(parametersTableName, accountsItemId); + const previousOrganizationalUnits = await loadOrganizations(parametersTableName, organizationsItemId); const organizationAdminRole = config['global-options']['organization-admin-role']; const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole, organizations); @@ -294,20 +296,30 @@ function updateAccountConfig(accountConfig: any, accountInfo: UpdateAccountOutpu } return accountConfig; } -async function loadAccounts(accountsSecretId: string): Promise { - const secret = await secrets.getSecret(accountsSecretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${accountsSecretId}"`); +async function loadAccounts(tableName: string, itemId: string): Promise { + let index = 0; + const accounts: Account[] = []; + while (true) { + const item = await dynamoDB.getItem(tableName, `${itemId}/${index}`); + if (index === 0 && !item.Item) { + throw new Error(`Cannot find parameter with ID "${itemId}"`); + } + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; } - return JSON.parse(secret.SecretString!); + return accounts; } -async function loadOrganizations(organizationsSecretId: string): Promise { - const secret = await secrets.getSecret(organizationsSecretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${organizationsSecretId}"`); +async function loadOrganizations(tableName: string, itemId: string): Promise { + const organizations = await dynamoDB.getItem(tableName, itemId); + + if (!organizations.Item) { + throw new Error(`Cannot find parameter with ID "${itemId}"`); } - return JSON.parse(secret.SecretString!); + return JSON.parse(organizations.Item.value.S!); } interface UpdateAccountsOutput { diff --git a/src/deployments/cdk/src/utils/accounts.ts b/src/deployments/cdk/src/utils/accounts.ts index cc9593a77..bc7509312 100644 --- a/src/deployments/cdk/src/utils/accounts.ts +++ b/src/deployments/cdk/src/utils/accounts.ts @@ -1,7 +1,7 @@ -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import * as fs from 'fs'; import * as path from 'path'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export { Account, getAccountId, getAccountArn } from '@aws-accelerator/common-outputs/src/accounts'; @@ -15,14 +15,29 @@ export async function loadAccounts(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.ACCOUNTS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "ACCOUNTS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const accountsItemId = process.env.ACCOUNTS_ITEM_ID; + if (!accountsItemId) { + throw new Error(`The environment variable "ACCOUNTS_ITEM_ID" needs to be set`); + } + + let index = 0; + const accounts: Account[] = []; + while (true) { + const item = await new DynamoDB().getItem(tableName, `${accountsItemId}/${index}`); + if (index === 0 && !item.Item) { + throw new Error(`Cannot find parameter with ID "${accountsItemId}"`); + } + + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; } - return JSON.parse(secret.SecretString!); + return accounts; } diff --git a/src/deployments/cdk/src/utils/limits.ts b/src/deployments/cdk/src/utils/limits.ts index db078aab6..d808fb773 100644 --- a/src/deployments/cdk/src/utils/limits.ts +++ b/src/deployments/cdk/src/utils/limits.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; @@ -17,16 +17,21 @@ export async function loadLimits(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.LIMITS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "LIMITS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const limitsItemId = process.env.LIMITS_ITEM_ID; + if (!limitsItemId) { + throw new Error(`The environment variable "LIMITS_ITEM_ID" needs to be set`); + } + + const limits = await new DynamoDB().getItem(tableName, limitsItemId); + if (!limits.Item) { + throw new Error(`Cannot find value with Item ID "${limitsItemId}"`); } - return JSON.parse(secret.SecretString!); + return JSON.parse(limits.Item.value.S!); } export function tryGetQuotaByAccountAndLimit( diff --git a/src/deployments/cdk/src/utils/organizations.ts b/src/deployments/cdk/src/utils/organizations.ts index 48ffb735e..12dbd2d92 100644 --- a/src/deployments/cdk/src/utils/organizations.ts +++ b/src/deployments/cdk/src/utils/organizations.ts @@ -1,7 +1,7 @@ -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import * as fs from 'fs'; import * as path from 'path'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export async function loadOrganizations(): Promise { if (process.env.CONFIG_MODE === 'development') { @@ -13,14 +13,19 @@ export async function loadOrganizations(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.ORGANIZATIONS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "ORGANIZATIONS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const organizationsItemId = process.env.ORGANIZATIONS_ITEM_ID; + if (!organizationsItemId) { + throw new Error(`The environment variable "ORGANIZATIONS_ITEM_ID" needs to be set`); + } + + const organizations = await new DynamoDB().getItem(tableName, organizationsItemId); + if (!organizations.Item) { + throw new Error(`Cannot find value with Item ID "${organizationsItemId}"`); } - return JSON.parse(secret.SecretString!); + return JSON.parse(organizations.Item.value.S!); } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts new file mode 100644 index 000000000..f713345ab --- /dev/null +++ b/src/lib/common/src/aws/dynamodb.ts @@ -0,0 +1,45 @@ +import aws from './aws-client'; +import * as dynamodb from 'aws-sdk/clients/dynamodb'; +import { throttlingBackOff } from './backoff'; + +export class DynamoDB { + private readonly client: aws.DynamoDB; + + constructor(credentials?: aws.Credentials) { + this.client = new aws.DynamoDB({ + credentials, + }); + } + + async createTable(props: dynamodb.CreateTableInput): Promise { + throttlingBackOff(() => this.client.createTable(props).promise()); + } + + async putItem(tableName: string, itemId: string, attributeValue: string): Promise { + const props = { + TableName: tableName, + Item: { + id: { S: itemId }, + value: { S: attributeValue }, + }, + }; + console.log('dynamodb putItem props', props); + throttlingBackOff(() => this.client.putItem(props).promise()); + } + + async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { + throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + } + + async scanTable(props: dynamodb.ScanInput): Promise { + return throttlingBackOff(() => this.client.scan(props).promise()); + } + + async getItem(tableName: string, itemId: string): Promise { + const props = { + TableName: `${tableName}`, + Key: { id: { S: `${itemId}` } }, + }; + return throttlingBackOff(() => this.client.getItem(props).promise()); + } +} From 6b0684954af295a4bf057ee19d7066178800e114 Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 1 Sep 2020 17:41:31 +0530 Subject: [PATCH 2/8] removed dynamodb service principal --- src/core/cdk/src/initial-setup.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 25ab61fdc..699fe419c 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -85,7 +85,6 @@ export namespace InitialSetup { new iam.ServicePrincipal('codebuild.amazonaws.com'), new iam.ServicePrincipal('lambda.amazonaws.com'), new iam.ServicePrincipal('events.amazonaws.com'), - new iam.ServicePrincipal('dynamodb.amazonaws.com'), ), managedPolicies: [iam.ManagedPolicy.fromAwsManagedPolicyName('AdministratorAccess')], maxSessionDuration: buildTimeout, From 02d8ff894c2e6237cdf6ea53e3240bae47a302ee Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 1 Sep 2020 17:55:48 +0530 Subject: [PATCH 3/8] added await --- src/core/runtime/src/load-organizations-step.ts | 2 -- src/lib/common/src/aws/dynamodb.ts | 11 +++++------ 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/core/runtime/src/load-organizations-step.ts b/src/core/runtime/src/load-organizations-step.ts index d5fafc386..9a10142cf 100644 --- a/src/core/runtime/src/load-organizations-step.ts +++ b/src/core/runtime/src/load-organizations-step.ts @@ -45,10 +45,8 @@ export const handler = async (input: LoadOrganizationsInput): Promise { - throttlingBackOff(() => this.client.createTable(props).promise()); + await throttlingBackOff(() => this.client.createTable(props).promise()); } async putItem(tableName: string, itemId: string, attributeValue: string): Promise { @@ -23,16 +23,15 @@ export class DynamoDB { value: { S: attributeValue }, }, }; - console.log('dynamodb putItem props', props); - throttlingBackOff(() => this.client.putItem(props).promise()); + await throttlingBackOff(() => this.client.putItem(props).promise()); } async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { - throttlingBackOff(() => this.client.batchWriteItem(props).promise()); + await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); } async scanTable(props: dynamodb.ScanInput): Promise { - return throttlingBackOff(() => this.client.scan(props).promise()); + return await throttlingBackOff(() => this.client.scan(props).promise()); } async getItem(tableName: string, itemId: string): Promise { @@ -40,6 +39,6 @@ export class DynamoDB { TableName: `${tableName}`, Key: { id: { S: `${itemId}` } }, }; - return throttlingBackOff(() => this.client.getItem(props).promise()); + return await throttlingBackOff(() => this.client.getItem(props).promise()); } } From 71f4f76ab0397b9678f16858570d2f1bd4e5ade6 Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 1 Sep 2020 19:05:40 +0530 Subject: [PATCH 4/8] removed throwing exception --- src/core/runtime/src/ou-validation.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/core/runtime/src/ou-validation.ts b/src/core/runtime/src/ou-validation.ts index fd3604cdb..0b1566277 100644 --- a/src/core/runtime/src/ou-validation.ts +++ b/src/core/runtime/src/ou-validation.ts @@ -301,9 +301,6 @@ async function loadAccounts(tableName: string, itemId: string): Promise { + const organizationalUnits: ConfigOrganizationalUnit[] = []; const organizations = await dynamoDB.getItem(tableName, itemId); - if (!organizations.Item) { - throw new Error(`Cannot find parameter with ID "${itemId}"`); + return organizationalUnits; } return JSON.parse(organizations.Item.value.S!); } From 519d7bdafaf50c7131877b6201e6020522f9bf7b Mon Sep 17 00:00:00 2001 From: nachundu Date: Tue, 1 Sep 2020 21:21:26 +0530 Subject: [PATCH 5/8] removing existing account items from dynamodb --- src/core/runtime/src/load-accounts-step.ts | 11 ++++++++++- src/lib/common/src/aws/dynamodb.ts | 8 ++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index 7549157fe..b5a695f94 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -75,9 +75,18 @@ export const handler = async (input: LoadAccountsInput): Promise this.client.getItem(props).promise()); } + + async deleteItem(tableName: string, itemId: string): Promise { + const props = { + TableName: `${tableName}`, + Key: { id: { S: `${itemId}` } }, + }; + await throttlingBackOff(() => this.client.deleteItem(props).promise()); + } } From 219d6cf2879150ddb7e7200e3b644c18c4230d47 Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 2 Sep 2020 14:57:03 +0530 Subject: [PATCH 6/8] deleted existing account items --- src/core/cdk/src/initial-setup.ts | 13 ++++++- src/core/runtime/src/load-accounts-step.ts | 27 +++++++++----- src/core/runtime/src/load-limits-step.ts | 3 +- .../runtime/src/load-organizations-step.ts | 3 +- src/core/runtime/src/ou-validation.ts | 5 ++- .../runtime/src/utils/dynamodb-requests.ts | 37 +++++++++++++++++++ src/deployments/cdk/src/utils/accounts.ts | 6 ++- src/deployments/cdk/src/utils/limits.ts | 7 +++- .../cdk/src/utils/organizations.ts | 7 +++- src/lib/common/src/aws/dynamodb.ts | 31 +++++----------- 10 files changed, 100 insertions(+), 39 deletions(-) create mode 100644 src/core/runtime/src/utils/dynamodb-requests.ts diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 699fe419c..3615b0d0e 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -68,9 +68,19 @@ export namespace InitialSetup { const stack = cdk.Stack.of(this); + const accountItemsCountSecret = new secrets.Secret(this, 'AccountItemsCount', { + secretName: 'accelerator/account-items-count', + description: 'This secret contains the information about account items count in dynamodb table.', + }); + setSecretValue(accountItemsCountSecret, '0'); + const parametersTable = new dynamodb.Table(this, 'ParametersTable', { + tableName: createName({ + name: 'Parameters', + suffixLength: 0, + }), partitionKey: { name: 'id', type: dynamodb.AttributeType.STRING }, - tableName: `${props.acceleratorPrefix}Parameters`, + encryption: dynamodb.TableEncryption.DEFAULT, }); // This is the maximum time before a build times out @@ -308,6 +318,7 @@ export namespace InitialSetup { functionPayload: { parametersTableName: parametersTable.tableName, itemId: 'accounts', + accountItemsCountSecretId: accountItemsCountSecret.secretArn, 'configuration.$': '$.configuration', }, resultPath: '$', diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index b5a695f94..cc0546837 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,10 +1,13 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; +import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests'; export interface LoadAccountsInput { + accountItemsCountSecretId: string; parametersTableName: string; itemId: string; configuration: LoadConfigurationOutput; @@ -17,12 +20,13 @@ export interface LoadAccountsOutput { } const dynamoDB = new DynamoDB(); +const secrets = new SecretsManager(); export const handler = async (input: LoadAccountsInput): Promise => { console.log(`Loading accounts...`); console.log(JSON.stringify(input, null, 2)); - const { parametersTableName, configuration, itemId } = input; + const { parametersTableName, configuration, itemId, accountItemsCountSecretId } = input; // The first step is to load all the execution roles const organizations = new Organizations(); @@ -75,23 +79,26 @@ export const handler = async (input: LoadAccountsInput): Promise { } // Store the limits in the dynamodb - await dynamoDB.putItem(parametersTableName, itemId, JSON.stringify(limits, null, 2)); + await dynamoDB.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(limits, null, 2))); }; diff --git a/src/core/runtime/src/load-organizations-step.ts b/src/core/runtime/src/load-organizations-step.ts index 9a10142cf..1e56ea930 100644 --- a/src/core/runtime/src/load-organizations-step.ts +++ b/src/core/runtime/src/load-organizations-step.ts @@ -3,6 +3,7 @@ import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getUpdateItemInput } from './utils/dynamodb-requests'; export interface LoadOrganizationsInput extends LoadConfigurationInput { parametersTableName: string; @@ -46,7 +47,7 @@ export const handler = async (input: LoadOrganizationsInput): Promise { const organizationalUnits: ConfigOrganizationalUnit[] = []; - const organizations = await dynamoDB.getItem(tableName, itemId); + const organizations = await dynamoDB.getItem(getItemInput(tableName, itemId)); if (!organizations.Item) { return organizationalUnits; } diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts new file mode 100644 index 000000000..0eef237ca --- /dev/null +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -0,0 +1,37 @@ +import { DynamoDB } from 'aws-sdk'; + +interface ItemInput { + TableName: string; + Key: { [key: string]: { S: string } }; +} + +export const getItemInput = ( + tableName: string, + itemId: string, +): ItemInput => { + return { + TableName: tableName, + Key: { id: { S: itemId } }, + }; +}; + +export const getUpdateItemInput = ( + tableName: string, + itemId: string, + attributeValue: string, +): DynamoDB.UpdateItemInput => { + return { + TableName: tableName, + Key: { + id: { S: itemId }, + }, + ExpressionAttributeNames: { + "#a": "value", + }, + UpdateExpression: 'set #a = :a', + ExpressionAttributeValues: { + ":a": { S: attributeValue }, + } + }; +}; + diff --git a/src/deployments/cdk/src/utils/accounts.ts b/src/deployments/cdk/src/utils/accounts.ts index bc7509312..fcfa8b8c5 100644 --- a/src/deployments/cdk/src/utils/accounts.ts +++ b/src/deployments/cdk/src/utils/accounts.ts @@ -28,7 +28,11 @@ export async function loadAccounts(): Promise { let index = 0; const accounts: Account[] = []; while (true) { - const item = await new DynamoDB().getItem(tableName, `${accountsItemId}/${index}`); + const itemsInput = { + TableName: tableName, + Key: { id: { S: `${accountsItemId}/${index}` } }, + }; + const item = await new DynamoDB().getItem(itemsInput); if (index === 0 && !item.Item) { throw new Error(`Cannot find parameter with ID "${accountsItemId}"`); } diff --git a/src/deployments/cdk/src/utils/limits.ts b/src/deployments/cdk/src/utils/limits.ts index d808fb773..30068b85b 100644 --- a/src/deployments/cdk/src/utils/limits.ts +++ b/src/deployments/cdk/src/utils/limits.ts @@ -27,7 +27,12 @@ export async function loadLimits(): Promise { throw new Error(`The environment variable "LIMITS_ITEM_ID" needs to be set`); } - const limits = await new DynamoDB().getItem(tableName, limitsItemId); + const itemsInput = { + TableName: tableName, + Key: { id: { S: limitsItemId } }, + }; + + const limits = await new DynamoDB().getItem(itemsInput); if (!limits.Item) { throw new Error(`Cannot find value with Item ID "${limitsItemId}"`); } diff --git a/src/deployments/cdk/src/utils/organizations.ts b/src/deployments/cdk/src/utils/organizations.ts index 12dbd2d92..683ac64de 100644 --- a/src/deployments/cdk/src/utils/organizations.ts +++ b/src/deployments/cdk/src/utils/organizations.ts @@ -23,7 +23,12 @@ export async function loadOrganizations(): Promise { throw new Error(`The environment variable "ORGANIZATIONS_ITEM_ID" needs to be set`); } - const organizations = await new DynamoDB().getItem(tableName, organizationsItemId); + const itemsInput = { + TableName: tableName, + Key: { id: { S: organizationsItemId } }, + }; + + const organizations = await new DynamoDB().getItem(itemsInput); if (!organizations.Item) { throw new Error(`Cannot find value with Item ID "${organizationsItemId}"`); } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index bdc93f20b..ec569a9bd 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -15,17 +15,6 @@ export class DynamoDB { await throttlingBackOff(() => this.client.createTable(props).promise()); } - async putItem(tableName: string, itemId: string, attributeValue: string): Promise { - const props = { - TableName: tableName, - Item: { - id: { S: itemId }, - value: { S: attributeValue }, - }, - }; - await throttlingBackOff(() => this.client.putItem(props).promise()); - } - async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); } @@ -34,19 +23,19 @@ export class DynamoDB { return await throttlingBackOff(() => this.client.scan(props).promise()); } - async getItem(tableName: string, itemId: string): Promise { - const props = { - TableName: `${tableName}`, - Key: { id: { S: `${itemId}` } }, - }; + async putItem(props: dynamodb.PutItemInput): Promise { + await throttlingBackOff(() => this.client.putItem(props).promise()); + } + + async getItem(props: dynamodb.GetItemInput): Promise { return await throttlingBackOff(() => this.client.getItem(props).promise()); } - async deleteItem(tableName: string, itemId: string): Promise { - const props = { - TableName: `${tableName}`, - Key: { id: { S: `${itemId}` } }, - }; + async deleteItem(props: dynamodb.DeleteItemInput): Promise { await throttlingBackOff(() => this.client.deleteItem(props).promise()); } + + async updateItem(props: dynamodb.UpdateItemInput): Promise { + await throttlingBackOff(() => this.client.updateItem(props).promise()); + } } From f81c6391fbbb0f672a1e34c71cdb4fa50c99e09d Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 2 Sep 2020 15:38:11 +0530 Subject: [PATCH 7/8] fixed lint issues --- src/core/runtime/src/load-accounts-step.ts | 12 +++-- src/core/runtime/src/ou-validation.ts | 6 +-- .../runtime/src/utils/dynamodb-requests.ts | 50 +++++++++---------- src/deployments/cdk/src/utils/limits.ts | 2 +- src/lib/common/src/aws/dynamodb.ts | 4 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index cc0546837..bacc869d7 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -35,8 +35,10 @@ export const handler = async (input: LoadAccountsInput): Promise - Array.from({ length: Math.ceil(accounts.length / size) }, (v, i) => accounts.slice(i * size, i * size + size)); + const chunk = (totalAccounts: Account[], size: number) => + Array.from({ length: Math.ceil(totalAccounts.length / size) }, (v, i) => + totalAccounts.slice(i * size, i * size + size), + ); for (const accountConfig of configuration.accounts) { let organizationAccount; @@ -90,8 +92,10 @@ export const handler = async (input: LoadAccountsInput): Promise { const organizationalUnits: ConfigOrganizationalUnit[] = []; - const organizations = await dynamoDB.getItem(getItemInput(tableName, itemId)); - if (!organizations.Item) { + const organizationsOutput = await dynamoDB.getItem(getItemInput(tableName, itemId)); + if (!organizationsOutput.Item) { return organizationalUnits; } - return JSON.parse(organizations.Item.value.S!); + return JSON.parse(organizationsOutput.Item.value.S!); } interface UpdateAccountsOutput { diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts index 0eef237ca..1296ff033 100644 --- a/src/core/runtime/src/utils/dynamodb-requests.ts +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -1,37 +1,33 @@ import { DynamoDB } from 'aws-sdk'; interface ItemInput { - TableName: string; - Key: { [key: string]: { S: string } }; + TableName: string; + Key: { [key: string]: { S: string } }; } -export const getItemInput = ( - tableName: string, - itemId: string, -): ItemInput => { - return { - TableName: tableName, - Key: { id: { S: itemId } }, - }; +export const getItemInput = (tableName: string, itemId: string): ItemInput => { + return { + TableName: tableName, + Key: { id: { S: itemId } }, + }; }; export const getUpdateItemInput = ( - tableName: string, - itemId: string, - attributeValue: string, + tableName: string, + itemId: string, + attributeValue: string, ): DynamoDB.UpdateItemInput => { - return { - TableName: tableName, - Key: { - id: { S: itemId }, - }, - ExpressionAttributeNames: { - "#a": "value", - }, - UpdateExpression: 'set #a = :a', - ExpressionAttributeValues: { - ":a": { S: attributeValue }, - } - }; + return { + TableName: tableName, + Key: { + id: { S: itemId }, + }, + ExpressionAttributeNames: { + '#a': 'value', + }, + UpdateExpression: 'set #a = :a', + ExpressionAttributeValues: { + ':a': { S: attributeValue }, + }, + }; }; - diff --git a/src/deployments/cdk/src/utils/limits.ts b/src/deployments/cdk/src/utils/limits.ts index 30068b85b..ffc1a26c2 100644 --- a/src/deployments/cdk/src/utils/limits.ts +++ b/src/deployments/cdk/src/utils/limits.ts @@ -31,7 +31,7 @@ export async function loadLimits(): Promise { TableName: tableName, Key: { id: { S: limitsItemId } }, }; - + const limits = await new DynamoDB().getItem(itemsInput); if (!limits.Item) { throw new Error(`Cannot find value with Item ID "${limitsItemId}"`); diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index ec569a9bd..bc3d57f02 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -20,7 +20,7 @@ export class DynamoDB { } async scanTable(props: dynamodb.ScanInput): Promise { - return await throttlingBackOff(() => this.client.scan(props).promise()); + return throttlingBackOff(() => this.client.scan(props).promise()); } async putItem(props: dynamodb.PutItemInput): Promise { @@ -28,7 +28,7 @@ export class DynamoDB { } async getItem(props: dynamodb.GetItemInput): Promise { - return await throttlingBackOff(() => this.client.getItem(props).promise()); + return throttlingBackOff(() => this.client.getItem(props).promise()); } async deleteItem(props: dynamodb.DeleteItemInput): Promise { From 9f388a93caf54a167422631d586980be438cd8a3 Mon Sep 17 00:00:00 2001 From: nachundu Date: Wed, 2 Sep 2020 18:32:21 +0530 Subject: [PATCH 8/8] moved accounts count to dynamodb --- src/core/cdk/src/initial-setup.ts | 8 +------- src/core/runtime/src/load-accounts-step.ts | 17 +++++++---------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index 234528946..42a8bf153 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -69,12 +69,6 @@ export namespace InitialSetup { const stack = cdk.Stack.of(this); - const accountItemsCountSecret = new secrets.Secret(this, 'AccountItemsCount', { - secretName: 'accelerator/account-items-count', - description: 'This secret contains the information about account items count in dynamodb table.', - }); - setSecretValue(accountItemsCountSecret, '0'); - const parametersTable = new dynamodb.Table(this, 'ParametersTable', { tableName: createName({ name: 'Parameters', @@ -331,7 +325,7 @@ export namespace InitialSetup { functionPayload: { parametersTableName: parametersTable.tableName, itemId: 'accounts', - accountItemsCountSecretId: accountItemsCountSecret.secretArn, + accountsItemsCountId: 'accounts-items-count', 'configuration.$': '$.configuration', }, resultPath: '$', diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index bacc869d7..5458f15db 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,5 +1,4 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; @@ -7,7 +6,7 @@ import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests'; export interface LoadAccountsInput { - accountItemsCountSecretId: string; + accountsItemsCountId: string; parametersTableName: string; itemId: string; configuration: LoadConfigurationOutput; @@ -20,13 +19,12 @@ export interface LoadAccountsOutput { } const dynamoDB = new DynamoDB(); -const secrets = new SecretsManager(); export const handler = async (input: LoadAccountsInput): Promise => { console.log(`Loading accounts...`); console.log(JSON.stringify(input, null, 2)); - const { parametersTableName, configuration, itemId, accountItemsCountSecretId } = input; + const { parametersTableName, configuration, itemId, accountsItemsCountId } = input; // The first step is to load all the execution roles const organizations = new Organizations(); @@ -81,8 +79,8 @@ export const handler = async (input: LoadAccountsInput): Promise