diff --git a/src/core/cdk/package.json b/src/core/cdk/package.json index 306a6d347..9d3214946 100644 --- a/src/core/cdk/package.json +++ b/src/core/cdk/package.json @@ -23,10 +23,13 @@ "typescript": "3.8.3" }, "dependencies": { + "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", + "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", "@aws-cdk/aws-cloudformation": "1.46.0", "@aws-cdk/aws-codebuild": "1.46.0", "@aws-cdk/aws-codepipeline": "1.46.0", "@aws-cdk/aws-codepipeline-actions": "1.46.0", + "@aws-cdk/aws-dynamodb": "1.46.0", "@aws-cdk/aws-events": "1.46.0", "@aws-cdk/aws-events-targets": "1.46.0", "@aws-cdk/aws-iam": "1.46.0", @@ -36,14 +39,11 @@ "@aws-cdk/aws-s3": "1.46.0", "@aws-cdk/aws-s3-assets": "1.46.0", "@aws-cdk/aws-s3-deployment": "1.46.0", + "@aws-cdk/aws-secretsmanager": "1.46.0", "@aws-cdk/aws-sns": "1.46.0", "@aws-cdk/aws-stepfunctions": "1.46.0", "@aws-cdk/aws-stepfunctions-tasks": "1.46.0", - "@aws-cdk/aws-secretsmanager": "1.46.0", - "@aws-cdk/aws-dynamodb": "1.46.0", "@aws-cdk/core": "1.46.0", - "@aws-accelerator/accelerator-runtime": "workspace:^0.0.1", - "@aws-accelerator/cdk-accelerator": "workspace:^0.0.1", "@types/cfn-response": "^1.0.3", "aws-sdk": "2.668.0", "cfn-response": "^1.0.1", diff --git a/src/core/cdk/src/initial-setup.ts b/src/core/cdk/src/initial-setup.ts index f00a697f4..42a8bf153 100644 --- a/src/core/cdk/src/initial-setup.ts +++ b/src/core/cdk/src/initial-setup.ts @@ -69,22 +69,14 @@ export namespace InitialSetup { const stack = cdk.Stack.of(this); - const accountsSecret = new secrets.Secret(this, 'Accounts', { - secretName: 'accelerator/accounts', - description: 'This secret contains the information about the accounts that are used for deployment.', - }); - setSecretValue(accountsSecret, '[]'); - - const limitsSecret = new secrets.Secret(this, 'Limits', { - secretName: 'accelerator/limits', - description: 'This secret contains a copy of the service limits of the Accelerator accounts.', - }); - - const organizationsSecret = new secrets.Secret(this, 'Organizations', { - secretName: 'accelerator/organizations', - description: 'This secret contains the information about the organizations that are used for deployment.', + const parametersTable = new dynamodb.Table(this, 'ParametersTable', { + tableName: createName({ + name: 'Parameters', + suffixLength: 0, + }), + partitionKey: { name: 'id', type: dynamodb.AttributeType.STRING }, + encryption: dynamodb.TableEncryption.DEFAULT, }); - setSecretValue(organizationsSecret, '[]'); const outputsTable = new dynamodb.Table(this, 'Outputs', { tableName: createName({ @@ -135,9 +127,10 @@ export namespace InitialSetup { ACCELERATOR_EXECUTION_ROLE_NAME: props.stateMachineExecutionRole, CDK_PLUGIN_ASSUME_ROLE_NAME: props.stateMachineExecutionRole, CDK_PLUGIN_ASSUME_ROLE_DURATION: `${buildTimeout.toSeconds()}`, - ACCOUNTS_SECRET_ID: accountsSecret.secretArn, - LIMITS_SECRET_ID: limitsSecret.secretArn, - ORGANIZATIONS_SECRET_ID: organizationsSecret.secretArn, + ACCOUNTS_ITEM_ID: 'accounts', + LIMITS_ITEM_ID: 'limits', + ORGANIZATIONS_ITEM_ID: 'organizations', + DYNAMODB_PARAMETERS_TABLE_NAME: parametersTable.tableName, }, }); @@ -314,7 +307,8 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - organizationsSecretId: organizationsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'organizations', configRepositoryName: props.configRepositoryName, 'configFilePath.$': '$.configuration.configFilePath', 'configCommitId.$': '$.configuration.configCommitId', @@ -329,7 +323,9 @@ export namespace InitialSetup { role: pipelineRole, }, functionPayload: { - accountsSecretId: accountsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'accounts', + accountsItemsCountId: 'accounts-items-count', 'configuration.$': '$.configuration', }, resultPath: '$', @@ -444,7 +440,8 @@ export namespace InitialSetup { 'configRepositoryName.$': '$.configRepositoryName', 'configFilePath.$': '$.configFilePath', 'configCommitId.$': '$.configCommitId', - limitsSecretId: limitsSecret.secretArn, + parametersTableName: parametersTable.tableName, + itemId: 'limits', assumeRoleName: props.stateMachineExecutionRole, 'accounts.$': '$.accounts', }, @@ -462,8 +459,9 @@ export namespace InitialSetup { 'configFilePath.$': '$.configuration.configFilePath', 'configCommitId.$': '$.configuration.configCommitId', acceleratorPrefix: props.acceleratorPrefix, - accountsSecretId: accountsSecret.secretArn, - organizationsSecretId: organizationsSecret.secretArn, + parametersTableName: parametersTable.tableName, + organizationsItemId: 'organizations', + accountsItemId: 'accounts', configBranch: props.configBranchName, 'configRootFilePath.$': '$.configuration.configRootFilePath', }, diff --git a/src/core/runtime/src/load-accounts-step.ts b/src/core/runtime/src/load-accounts-step.ts index 935ef61b9..5458f15db 100644 --- a/src/core/runtime/src/load-accounts-step.ts +++ b/src/core/runtime/src/load-accounts-step.ts @@ -1,11 +1,14 @@ import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import { LoadConfigurationOutput, ConfigurationOrganizationalUnit } from './load-configuration-step'; import { equalIgnoreCase } from '@aws-accelerator/common/src/util/common'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getItemInput, getUpdateItemInput } from './utils/dynamodb-requests'; export interface LoadAccountsInput { - accountsSecretId: string; + accountsItemsCountId: string; + parametersTableName: string; + itemId: string; configuration: LoadConfigurationOutput; } @@ -15,11 +18,13 @@ export interface LoadAccountsOutput { regions: string[]; } +const dynamoDB = new DynamoDB(); + export const handler = async (input: LoadAccountsInput): Promise => { console.log(`Loading accounts...`); console.log(JSON.stringify(input, null, 2)); - const { accountsSecretId, configuration } = input; + const { parametersTableName, configuration, itemId, accountsItemsCountId } = input; // The first step is to load all the execution roles const organizations = new Organizations(); @@ -27,6 +32,12 @@ export const handler = async (input: LoadAccountsInput): Promise account.Status === 'ACTIVE'); const accounts = []; + + const chunk = (totalAccounts: Account[], size: number) => + Array.from({ length: Math.ceil(totalAccounts.length / size) }, (v, i) => + totalAccounts.slice(i * size, i * size + size), + ); + for (const accountConfig of configuration.accounts) { let organizationAccount; organizationAccount = activeAccounts.find(a => { @@ -68,12 +79,26 @@ export const handler = async (input: LoadAccountsInput): Promise { console.log(`Loading limits...`); console.log(JSON.stringify(input, null, 2)); - const { configRepositoryName, configFilePath, limitsSecretId, accounts, assumeRoleName, configCommitId } = input; + const { + configRepositoryName, + configFilePath, + parametersTableName, + accounts, + assumeRoleName, + configCommitId, + itemId, + } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ @@ -147,10 +159,6 @@ export const handler = async (input: LoadLimitsInput) => { } } - // Store the limits in the secrets manager - const secrets = new SecretsManager(); - await secrets.putSecretValue({ - SecretId: limitsSecretId, - SecretString: JSON.stringify(limits, null, 2), - }); + // Store the limits in the dynamodb + await dynamoDB.updateItem(getUpdateItemInput(parametersTableName, itemId, JSON.stringify(limits, null, 2))); }; diff --git a/src/core/runtime/src/load-organizations-step.ts b/src/core/runtime/src/load-organizations-step.ts index a8c0beee0..1e56ea930 100644 --- a/src/core/runtime/src/load-organizations-step.ts +++ b/src/core/runtime/src/load-organizations-step.ts @@ -1,26 +1,28 @@ import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { LoadConfigurationInput } from './load-configuration-step'; import { loadAcceleratorConfig } from '@aws-accelerator/common-config/src/load'; import { Organizations } from '@aws-accelerator/common/src/aws/organizations'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; +import { getUpdateItemInput } from './utils/dynamodb-requests'; export interface LoadOrganizationsInput extends LoadConfigurationInput { - organizationsSecretId: string; + parametersTableName: string; + itemId: string; } export type LoadOrganizationsOutput = { organizationalUnits: OrganizationalUnit[]; }; -const secrets = new SecretsManager(); const organizations = new Organizations(); +const dynamoDB = new DynamoDB(); export const handler = async (input: LoadOrganizationsInput): Promise => { console.log('Load Organizations ...'); console.log(JSON.stringify(input, null, 2)); const organizationalUnits: OrganizationalUnit[] = []; - const { organizationsSecretId, configCommitId, configFilePath, configRepositoryName } = input; + const { configCommitId, configFilePath, configRepositoryName, parametersTableName, itemId } = input; // Retrieve Configuration from Code Commit with specific commitId const config = await loadAcceleratorConfig({ repositoryName: configRepositoryName, @@ -44,11 +46,8 @@ export const handler = async (input: LoadOrganizationsInput): Promise => { configRepositoryName, configCommitId, acceleratorPrefix, - accountsSecretId, - organizationsSecretId, + parametersTableName, + organizationsItemId, + accountsItemId, configBranch, configRootFilePath, } = input; @@ -60,8 +63,8 @@ export const handler = async (input: ValdationInput): Promise => { let rootConfig = getFormattedObject(rootConfigString, format); let config = previousConfig; - const previousAccounts = await loadAccounts(accountsSecretId); - const previousOrganizationalUnits = await loadOrganizations(organizationsSecretId); + const previousAccounts = await loadAccounts(parametersTableName, accountsItemId); + const previousOrganizationalUnits = await loadOrganizations(parametersTableName, organizationsItemId); const organizationAdminRole = config['global-options']['organization-admin-role']; const scps = new ServiceControlPolicy(acceleratorPrefix, organizationAdminRole, organizations); @@ -294,20 +297,27 @@ function updateAccountConfig(accountConfig: any, accountInfo: UpdateAccountOutpu } return accountConfig; } -async function loadAccounts(accountsSecretId: string): Promise { - const secret = await secrets.getSecret(accountsSecretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${accountsSecretId}"`); +async function loadAccounts(tableName: string, itemId: string): Promise { + let index = 0; + const accounts: Account[] = []; + while (true) { + const item = await dynamoDB.getItem(getItemInput(tableName, `${itemId}/${index}`)); + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; } - return JSON.parse(secret.SecretString!); + return accounts; } -async function loadOrganizations(organizationsSecretId: string): Promise { - const secret = await secrets.getSecret(organizationsSecretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${organizationsSecretId}"`); +async function loadOrganizations(tableName: string, itemId: string): Promise { + const organizationalUnits: ConfigOrganizationalUnit[] = []; + const organizationsOutput = await dynamoDB.getItem(getItemInput(tableName, itemId)); + if (!organizationsOutput.Item) { + return organizationalUnits; } - return JSON.parse(secret.SecretString!); + return JSON.parse(organizationsOutput.Item.value.S!); } interface UpdateAccountsOutput { diff --git a/src/core/runtime/src/utils/dynamodb-requests.ts b/src/core/runtime/src/utils/dynamodb-requests.ts new file mode 100644 index 000000000..1296ff033 --- /dev/null +++ b/src/core/runtime/src/utils/dynamodb-requests.ts @@ -0,0 +1,33 @@ +import { DynamoDB } from 'aws-sdk'; + +interface ItemInput { + TableName: string; + Key: { [key: string]: { S: string } }; +} + +export const getItemInput = (tableName: string, itemId: string): ItemInput => { + return { + TableName: tableName, + Key: { id: { S: itemId } }, + }; +}; + +export const getUpdateItemInput = ( + tableName: string, + itemId: string, + attributeValue: string, +): DynamoDB.UpdateItemInput => { + return { + TableName: tableName, + Key: { + id: { S: itemId }, + }, + ExpressionAttributeNames: { + '#a': 'value', + }, + UpdateExpression: 'set #a = :a', + ExpressionAttributeValues: { + ':a': { S: attributeValue }, + }, + }; +}; diff --git a/src/deployments/cdk/src/utils/accounts.ts b/src/deployments/cdk/src/utils/accounts.ts index cc9593a77..fcfa8b8c5 100644 --- a/src/deployments/cdk/src/utils/accounts.ts +++ b/src/deployments/cdk/src/utils/accounts.ts @@ -1,7 +1,7 @@ -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import { Account } from '@aws-accelerator/common-outputs/src/accounts'; import * as fs from 'fs'; import * as path from 'path'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export { Account, getAccountId, getAccountArn } from '@aws-accelerator/common-outputs/src/accounts'; @@ -15,14 +15,33 @@ export async function loadAccounts(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.ACCOUNTS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "ACCOUNTS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const accountsItemId = process.env.ACCOUNTS_ITEM_ID; + if (!accountsItemId) { + throw new Error(`The environment variable "ACCOUNTS_ITEM_ID" needs to be set`); + } + + let index = 0; + const accounts: Account[] = []; + while (true) { + const itemsInput = { + TableName: tableName, + Key: { id: { S: `${accountsItemId}/${index}` } }, + }; + const item = await new DynamoDB().getItem(itemsInput); + if (index === 0 && !item.Item) { + throw new Error(`Cannot find parameter with ID "${accountsItemId}"`); + } + + if (!item.Item) { + break; + } + accounts.push(...JSON.parse(item.Item.value.S!)); + index++; } - return JSON.parse(secret.SecretString!); + return accounts; } diff --git a/src/deployments/cdk/src/utils/limits.ts b/src/deployments/cdk/src/utils/limits.ts index db078aab6..ffc1a26c2 100644 --- a/src/deployments/cdk/src/utils/limits.ts +++ b/src/deployments/cdk/src/utils/limits.ts @@ -1,7 +1,7 @@ import * as fs from 'fs'; import * as path from 'path'; import { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export { Limit, LimitOutput } from '@aws-accelerator/common-outputs/src/limits'; @@ -17,16 +17,26 @@ export async function loadLimits(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.LIMITS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "LIMITS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const limitsItemId = process.env.LIMITS_ITEM_ID; + if (!limitsItemId) { + throw new Error(`The environment variable "LIMITS_ITEM_ID" needs to be set`); + } + + const itemsInput = { + TableName: tableName, + Key: { id: { S: limitsItemId } }, + }; + + const limits = await new DynamoDB().getItem(itemsInput); + if (!limits.Item) { + throw new Error(`Cannot find value with Item ID "${limitsItemId}"`); } - return JSON.parse(secret.SecretString!); + return JSON.parse(limits.Item.value.S!); } export function tryGetQuotaByAccountAndLimit( diff --git a/src/deployments/cdk/src/utils/organizations.ts b/src/deployments/cdk/src/utils/organizations.ts index 48ffb735e..683ac64de 100644 --- a/src/deployments/cdk/src/utils/organizations.ts +++ b/src/deployments/cdk/src/utils/organizations.ts @@ -1,7 +1,7 @@ -import { SecretsManager } from '@aws-accelerator/common/src/aws/secrets-manager'; import * as fs from 'fs'; import * as path from 'path'; import { OrganizationalUnit } from '@aws-accelerator/common-outputs/src/organizations'; +import { DynamoDB } from '@aws-accelerator/common/src/aws/dynamodb'; export async function loadOrganizations(): Promise { if (process.env.CONFIG_MODE === 'development') { @@ -13,14 +13,24 @@ export async function loadOrganizations(): Promise { return JSON.parse(contents.toString()); } - const secretId = process.env.ORGANIZATIONS_SECRET_ID; - if (!secretId) { - throw new Error(`The environment variable "ORGANIZATIONS_SECRET_ID" needs to be set`); + const tableName = process.env.DYNAMODB_PARAMETERS_TABLE_NAME; + if (!tableName) { + throw new Error(`The environment variable "DYNAMODB_PARAMETERS_TABLE_NAME" needs to be set`); } - const secrets = new SecretsManager(); - const secret = await secrets.getSecret(secretId); - if (!secret) { - throw new Error(`Cannot find secret with ID "${secretId}"`); + + const organizationsItemId = process.env.ORGANIZATIONS_ITEM_ID; + if (!organizationsItemId) { + throw new Error(`The environment variable "ORGANIZATIONS_ITEM_ID" needs to be set`); + } + + const itemsInput = { + TableName: tableName, + Key: { id: { S: organizationsItemId } }, + }; + + const organizations = await new DynamoDB().getItem(itemsInput); + if (!organizations.Item) { + throw new Error(`Cannot find value with Item ID "${organizationsItemId}"`); } - return JSON.parse(secret.SecretString!); + return JSON.parse(organizations.Item.value.S!); } diff --git a/src/lib/common/src/aws/dynamodb.ts b/src/lib/common/src/aws/dynamodb.ts index 87325c0fe..9b582181a 100644 --- a/src/lib/common/src/aws/dynamodb.ts +++ b/src/lib/common/src/aws/dynamodb.ts @@ -15,10 +15,6 @@ export class DynamoDB { await throttlingBackOff(() => this.client.createTable(props).promise()); } - async putItem(props: dynamodb.PutItemInput): Promise { - await throttlingBackOff(() => this.client.putItem(props).promise()); - } - async batchWriteItem(props: dynamodb.BatchWriteItemInput): Promise { await throttlingBackOff(() => this.client.batchWriteItem(props).promise()); } @@ -27,7 +23,19 @@ export class DynamoDB { return throttlingBackOff(() => this.client.scan(props).promise()); } + async putItem(props: dynamodb.PutItemInput): Promise { + await throttlingBackOff(() => this.client.putItem(props).promise()); + } + + async getItem(props: dynamodb.GetItemInput): Promise { + return throttlingBackOff(() => this.client.getItem(props).promise()); + } + async deleteItem(props: dynamodb.DeleteItemInput): Promise { await throttlingBackOff(() => this.client.deleteItem(props).promise()); } + + async updateItem(props: dynamodb.UpdateItemInput): Promise { + await throttlingBackOff(() => this.client.updateItem(props).promise()); + } }