Skip to content

Commit

Permalink
Merge ad0d738 into 4d34495
Browse files Browse the repository at this point in the history
  • Loading branch information
@Alexandelphi
Alexandelphi committed Dec 14, 2020
2 parents 4d34495 + ad0d738 commit 0095b5c
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 25 deletions.
10 changes: 9 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,19 @@ plugins:
```yaml
custom:
globalTables:
version: v1 # optional, please use 'v1' for 2017.11.29 or 'v2' for 2019.11.21 version creation
regions: # list of regions in which you want to set up global tables
- region-1
- region-2
createStack: false # optional flag, when set to false will not deploy the stack in new region(s) and will create the tables using AWS SDK.
```

_NOTE_: When creating global tables with `createStack: false`, any update the source table config is not replicated to global tables.
_NOTE_:
1. When creating global tables with `createStack: false`, any update the source table config is not replicated to global tables.
2. `version` field is backward compatible and not required (the field can be absent).
If you want to use Global Table (Version 2019.11.21), please use `version: v2`.
Also, we don't recommend using `v2` over `v1` in your existing project. The plugin doesn't support updating from `v1` to `v2`.
More details about Global Tables you can find in the following link: [AWS DynamoDB Global Tables](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GlobalTables.html)

## Revisions
* 2.0.0
Expand All @@ -36,4 +42,6 @@ _NOTE_: When creating global tables with `createStack: false`, any update the so
- Added unit-tests
* 2.1.0
- Added support for PAY_PER_REQUEST billing mode for `createStack: true` mode
* 3.1.0
- Added support to create a new version of Global Table (Version 2019.11.21)

6 changes: 3 additions & 3 deletions package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "serverless-create-global-dynamodb-table",
"version": "3.0.0",
"version": "3.1.0",
"description": "serverless plugin that would create global dynamodb tables for specified tables",
"main": "src/index.js",
"scripts": {
Expand Down Expand Up @@ -34,9 +34,9 @@
"sinon": "^7.4.2"
},
"dependencies": {
"aws-sdk": "^2.224.1",
"aws-sdk": "^2.804.0",
"chalk": "^2.3.2",
"lodash.get": "^4.4.2"
"lodash": "^4.17.11"
},
"husky": {
"hooks": {
Expand Down
64 changes: 53 additions & 11 deletions src/helper.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
const AWS = require('aws-sdk')
const chalk = require('chalk')
const get = require('lodash.get');
const get = require('lodash/get');

const WRITEAUOTSCALINGPOLICY = 'WriteAutoScalingPolicy';
const READAUOTSCALINGPOLICY = 'ReadAutoScalingPolicy';
Expand Down Expand Up @@ -184,17 +184,29 @@ const createNewTableAndSetScalingPolicy = async function createNewTableAndSetSca
* @param {string} region AWS region in which source table exists
* @param {Array} newRegions List of regions in which global tables needs to be created
* @param {string} tableName Dynamodb table name
* @param {string} version It's version of global table needs to be setup
* @param {Object} cli Serverless cli object
* @returns {Object} List of regions in which gloabl table needs to be created and flag indicating
* if some global table setup already exists.
*/
const getRegionsToCreateGlobalTablesIn = async function getRegionsToCreateGlobalTablesIn(
dynamodb, region, newRegions, tableName, cli
dynamodb, region, newRegions, tableName, version, cli
) {
try {
const resp = await dynamodb.describeGlobalTable({ GlobalTableName: tableName }).promise()
const regionsGlobalTableExists = resp.GlobalTableDescription.ReplicationGroup.map(rg => rg.RegionName);
const missingRegions = [region].concat(newRegions).filter(r => !regionsGlobalTableExists.includes(r));
let regionsGlobalTableExists = [];
let missingRegions = []
if (version === 'v2') {
let resp = await dynamodb.describeTable({ TableName: tableName }).promise()
if (resp.Table.Replicas !== undefined) {
regionsGlobalTableExists = resp.Table.Replicas.map(rg => rg.RegionName);
}
missingRegions = newRegions.filter(r => !regionsGlobalTableExists.includes(r));
} else {
let resp = await dynamodb.describeGlobalTable({ GlobalTableName: tableName }).promise()
regionsGlobalTableExists = resp.GlobalTableDescription.ReplicationGroup.map(rg => rg.RegionName);
missingRegions = [region].concat(newRegions).filter(r => !regionsGlobalTableExists.includes(r));
}

if (missingRegions.length === 0) {
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Global table ${tableName} already exists in all the specified regions. Skipping creation...`)}`)
return {missingRegions: [], addingNewRegions: false };
Expand All @@ -218,23 +230,23 @@ const getRegionsToCreateGlobalTablesIn = async function getRegionsToCreateGlobal
* @param {string} region AWS region in which source table exists
* @param {string} tableName Dymanodb table name
* @param {Array} newRegions List of regions in which global table needs to be setup
* @param {string} version It's version of global table needs to be setup
* @param {boolean} createStack flag indicating if the tables were created using cloudformation
* @param {Object} cli Serverless cli object
*/
const createGlobalTable = async function createGlobalTable(
appAutoScaling, dynamodb, creds, region, tableName, newRegions, createStack, cli
appAutoScaling, dynamodb, creds, region, tableName, newRegions, version, createStack, cli
) {

const { missingRegions: regionsToUpdate, addingNewRegions } = await module.exports.getRegionsToCreateGlobalTablesIn(
dynamodb, region, newRegions, tableName, cli
dynamodb, region, newRegions, tableName, version, cli
);

if (!regionsToUpdate.length) {
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Global table setup already in place.`)}`);
return;
}

if (!createStack) {
if (!createStack && version !== 'v2') {
const tableDef = await dynamodb.describeTable({ TableName: tableName }).promise()
const { ReadCapacityUnits, WriteCapacityUnits } = tableDef.Table.ProvisionedThroughput
const { GlobalSecondaryIndexes, LocalSecondaryIndexes } = tableDef.Table;
Expand Down Expand Up @@ -301,6 +313,16 @@ const createGlobalTable = async function createGlobalTable(
}));
}

if (version === 'v2') {
await module.exports.createGlobalTableV2(dynamodb, tableName, regionsToUpdate, cli);
} else {
await module.exports.createGlobalTableV1(dynamodb, tableName, region, regionsToUpdate, addingNewRegions, cli);
}

}

const createGlobalTableV1 = async function createGlobalTableV1(dynamodb, tableName, region, regionsToUpdate, addingNewRegions, cli) {
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Create global table setup (Version 2017.11.29) for ${tableName}...`)}`)
if (!addingNewRegions) {
const replicationGroup = [{ RegionName: region }]
regionsToUpdate.forEach(r => replicationGroup.push({ RegionName: r }))
Expand All @@ -310,7 +332,6 @@ const createGlobalTable = async function createGlobalTable(
ReplicationGroup: replicationGroup,
}
await dynamodb.createGlobalTable(param).promise()
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup for ${tableName}...`)}`)
} else {
const replicaUpdates = [];
regionsToUpdate.forEach(r => replicaUpdates.push({ Create:{ RegionName: r }}));
Expand All @@ -319,8 +340,25 @@ const createGlobalTable = async function createGlobalTable(
ReplicaUpdates: replicaUpdates,
}
await dynamodb.updateGlobalTable(param).promise();
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup for ${tableName}...`)}`)
}
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Created global table setup (Version 2017.11.29) for ${tableName}...`)}`)
}

const createGlobalTableV2 = async function createGlobalTableV2(dynamodb, tableName, regionsToUpdate, cli) {
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Create global table setup (Version 2019.11.21) for ${tableName}...`)}`)
for (const region of regionsToUpdate) {
const params = {
TableName: tableName,
ReplicaUpdates: [{ Create:{ RegionName: region }}],
}
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Wait for ${tableName} replication available...`)}`)
await dynamodb.waitFor('tableExists', {TableName: tableName}).promise(); // it's gonna wait for "Active" status
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`Start creating a replica for ${tableName} in ${region}`)}`)
await dynamodb.updateTable(params).promise();
await dynamodb.waitFor('tableExists', {TableName: tableName}).promise(); // it's gonna wait for "Active" status
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`The replica for ${tableName} in ${region} has been created successfully`)}`)
}
cli.consoleLog(`CreateGlobalTable: ${chalk.yellow(`The global table setup (Version 2019.11.21) for ${tableName} has been created successfully`)}`)
}

/**
Expand Down Expand Up @@ -392,6 +430,7 @@ const createGlobalDynamodbTable = async function createGlobalDynamodbTable(serve
region,
tableName,
globalTablesOptions.regions,
globalTablesOptions.version,
false,
cli
)
Expand All @@ -414,6 +453,7 @@ const createGlobalDynamodbTable = async function createGlobalDynamodbTable(serve
region,
tableName,
globalTablesOptions.regions,
globalTablesOptions.version,
true,
cli
)
Expand All @@ -428,6 +468,8 @@ module.exports = {
checkStackCreateUpdateStatus,
createGlobalDynamodbTable,
createGlobalTable,
createGlobalTableV1,
createGlobalTableV2,
createNewTableAndSetScalingPolicy,
createUpdateCfnStack,
getRegionsToCreateGlobalTablesIn,
Expand Down
76 changes: 66 additions & 10 deletions test/unittest/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,7 +414,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
'us-west-1',
'us-east-1'
]
describe('no global table exists', () => {
describe('no global table exists for v1', () => {
before(() => {
const error = new Error('forced error');
error.code = 'GlobalTableNotFoundException';
Expand All @@ -426,7 +426,26 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
dynamodb.describeGlobalTable.restore();
});
it ('should return all the new regions', async () => {
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
resp.missingRegions.should.have.length(2);
resp.missingRegions.should.eql(newRegions);
resp.addingNewRegions.should.eql(false);
});
});

describe('no global table exists for v2', () => {
before(() => {
const error = new Error('forced error');
error.code = 'GlobalTableNotFoundException';
sinon.stub(dynamodb, 'describeTable').returns({
promise: () => { return Promise.reject(error)}
});
});
after(() => {
dynamodb.describeTable.restore();
});
it ('should return all the new regions', async () => {
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v2', serverless.cli);
resp.missingRegions.should.have.length(2);
resp.missingRegions.should.eql(newRegions);
resp.addingNewRegions.should.eql(false);
Expand All @@ -446,7 +465,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
});
it ('should return all the new regions', async () => {
try {
await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
} catch (err) {
err.code.should.eql('RandomError');
}
Expand All @@ -471,7 +490,7 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
dynamodb.describeGlobalTable.restore();
});
it ('should return no region', async () => {
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
resp.missingRegions.should.have.length(0);
resp.addingNewRegions.should.eql(false);
});
Expand All @@ -494,7 +513,31 @@ describe('test getRegionsToCreateGlobalTablesIn function', () =>{
dynamodb.describeGlobalTable.restore();
});
it ('should return no region', async () => {
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', serverless.cli);
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v1', serverless.cli);
resp.missingRegions.should.have.length(1);
resp.missingRegions[0].should.eql('us-east-1');
resp.addingNewRegions.should.eql(true);
});
});

describe('global table exists in all the specified new regions for v2', () => {
before(() => {
sinon.stub(dynamodb, 'describeTable').returns({
promise: () => { return Promise.resolve({
Table: {
Replicas: [
{ RegionName: 'us-west-1'},
{ RegionName: 'us-west-2'}
]
}
})}
});
});
after(() => {
dynamodb.describeTable.restore();
});
it ('should return no region', async () => {
const resp = await plugin.getRegionsToCreateGlobalTablesIn(dynamodb, 'us-west-2', newRegions, 'test-table', 'v2', serverless.cli);
resp.missingRegions.should.have.length(1);
resp.missingRegions[0].should.eql('us-east-1');
resp.addingNewRegions.should.eql(true);
Expand Down Expand Up @@ -557,6 +600,12 @@ describe('test createGlobalTable function', () => {
sandbox.stub(dynamodb, 'updateGlobalTable').returns({
promise: () => { return Promise.resolve()}
});
sandbox.stub(dynamodb, 'updateTable').returns({
promise: () => { return Promise.resolve()}
});
sandbox.stub(dynamodb, 'waitFor').returns({
promise: () => { return Promise.resolve()}
});
});
afterEach(() => {
sandbox.restore();
Expand All @@ -568,12 +617,12 @@ describe('test createGlobalTable function', () => {
missingRegions: [],
addingNewRegions: false
}));
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1',true, serverless.cli);
sandbox.assert.notCalled(dynamodb.createGlobalTable);
});

it ('should create global table', async () => {
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', true, serverless.cli);
sandbox.assert.calledOnce(dynamodb.createGlobalTable);
});

Expand All @@ -583,14 +632,14 @@ describe('test createGlobalTable function', () => {
missingRegions: ['us-west-1'],
addingNewRegions: true
}));
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], true, serverless.cli);
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', true, serverless.cli);
sandbox.assert.notCalled(dynamodb.createGlobalTable);
sandbox.assert.calledOnce(dynamodb.updateGlobalTable);
});

context("when create stack is false", () => {
it ('should create the table with ProvisionedThroughput if billing mode is not PAY_PER_REQUEST', async () => {
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], false, serverless.cli);
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', false, serverless.cli);
sandbox.assert.calledOnce(dynamodb.describeTable);
plugin.createNewTableAndSetScalingPolicy.lastCall.args[2].should.eql({ AttributeDefinitions: {},
KeySchema: '',
Expand All @@ -612,7 +661,7 @@ describe('test createGlobalTable function', () => {

it ('should create the table without ProvisionedThroughput if billing mode is PAY_PER_REQUEST', async () => {
stubbedTable.Table.BillingModeSummary = { BillingMode: "PAY_PER_REQUEST" };
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], false, serverless.cli);
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v1', false, serverless.cli);
sandbox.assert.calledOnce(dynamodb.describeTable);
plugin.createNewTableAndSetScalingPolicy.lastCall.args[2].should.eql({ AttributeDefinitions: {},
KeySchema: '',
Expand All @@ -628,6 +677,13 @@ describe('test createGlobalTable function', () => {
BillingMode: 'PAY_PER_REQUEST',
Tags: {} });
});

it ('should create the table by using v2 version', async () => {
stubbedTable.Table.BillingModeSummary = { BillingMode: "PAY_PER_REQUEST" };
await plugin.createGlobalTable(aas, dynamodb, serverless.getProvider().getCredentials(), 'us-west-2', 'test-table', ['us-east-2'], 'v2', false, serverless.cli);
sandbox.assert.calledOnce(dynamodb.updateTable);
sandbox.assert.calledTwice(dynamodb.waitFor);
});
})
});

Expand Down

0 comments on commit 0095b5c

Please sign in to comment.