diff --git a/.env-example b/.env-example new file mode 100644 index 0000000..114ac9d --- /dev/null +++ b/.env-example @@ -0,0 +1,2 @@ +DYNAMO_ENDPOINT=http://localhost:8000 +REGION=ap-northeast-1 diff --git a/.gitignore b/.gitignore index f5a8916..df36331 100644 --- a/.gitignore +++ b/.gitignore @@ -109,3 +109,6 @@ dist .dist/ +src/migrate/data/* +!src/migrate/data/keep.me + diff --git a/README.md b/README.md new file mode 100644 index 0000000..ae153bd --- /dev/null +++ b/README.md @@ -0,0 +1,38 @@ +# Dynamodb Generic Repository + +## Install + +### Deploying DynamoDB Locally on Your Computer + +- Docker + + ```shell script + docker run -p 8000:8000 -it --rm instructure/dynamo-local-admin + ``` + + +- Amazon DynamoDB is provided as an executable .jar file + + [Documentation](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html) + + +## Migrate database + +### Create tables + +```shell script +ts-node ./src/migrate/create-tables.ts +``` + +### Load Sample Data + +[moviedata.zip](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/samples/moviedata.zip) + +```shell script +ts-node ./src/migrate/load-simple-movies.ts +``` + + + + + diff --git a/package-lock.json b/package-lock.json index 77e9189..e4d832b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -24,6 +24,21 @@ "js-tokens": "^4.0.0" } }, + "@types/dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/@types/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-ylSC9GhfRH7m1EUXBXofhgx4lUWmFeQDINW5oLuS+gxWdfUeW4zJdeVTYVkexEW+e2VUvlZR2kGnGGipAWR7kw==", + "dev": true, + "requires": { + "dotenv": "*" + } + }, + "@types/node": { + "version": "13.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.4.tgz", + "integrity": "sha512-oVeL12C6gQS/GAExndigSaLxTrKpQPxewx9bOcwfvJiJge4rr7wNaph4J+ns5hrmIV2as5qxqN8YKthn9qh0jw==", + "dev": true + }, "@types/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", @@ -66,12 +81,33 @@ "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", "dev": true }, + "aws-sdk": { + "version": "2.624.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.624.0.tgz", + "integrity": "sha512-6MhbdND7A5lEBiNSZ/HLwhKgrysmwTy6C47H7vfuVnY25hDkIND3C0PLqeRyskUqxv0RqsiAB4kqiMtpE08IGA==", + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", "dev": true }, + "base64-js": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", + "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -82,6 +118,16 @@ "concat-map": "0.0.1" } }, + "buffer": { + "version": "4.9.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", @@ -185,6 +231,11 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==" + }, "dynamic-dedupe": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", @@ -221,6 +272,11 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, "filewatcher": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/filewatcher/-/filewatcher-3.0.1.tgz", @@ -290,6 +346,11 @@ "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==", "dev": true }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "indent-string": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-2.1.0.tgz", @@ -339,12 +400,22 @@ "integrity": "sha1-HxbkqiKwTRM2tmGIpmrzxgDDpm0=", "dev": true }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, "isexe": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=", "dev": true }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -548,6 +619,16 @@ "pinkie": "^2.0.0" } }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, "read-pkg": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz", @@ -606,6 +687,11 @@ "glob": "^7.1.3" } }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -818,6 +904,20 @@ "integrity": "sha512-EgOVgL/4xfVrCMbhYKUQTdF37SQn4Iw73H5BgCrF1Abdun7Kwy/QZsE/ssAy0y4LxBbvua3PIbFsbRczWWnDdQ==", "dev": true }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, "validate-npm-package-license": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", @@ -843,6 +943,20 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", "dev": true }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + }, "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 6c694f2..562c38a 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,14 @@ "author": "hoangdv (https://codetheworld.io/)", "license": "ISC", "devDependencies": { + "@types/dotenv": "^8.2.0", + "@types/node": "^13.7.4", "ts-node-dev": "^1.0.0-pre.44", "tslint": "^6.0.0", "typescript": "^3.8.2" + }, + "dependencies": { + "aws-sdk": "^2.624.0", + "dotenv": "^8.2.0" } } diff --git a/src/config/index.ts b/src/config/index.ts new file mode 100644 index 0000000..5e13e52 --- /dev/null +++ b/src/config/index.ts @@ -0,0 +1,12 @@ +/** + * Created by SUN-ASTERISK\dinh.van.hoang on 2/24/20 + */ + +import dotenv from 'dotenv'; + +dotenv.config(); + +export const ENVIRONMENTS = { + DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT || 'http://localhost:8000', + REGION: process.env.REGION || 'local', +}; diff --git a/src/migrate/create-tables.ts b/src/migrate/create-tables.ts new file mode 100644 index 0000000..5a523a8 --- /dev/null +++ b/src/migrate/create-tables.ts @@ -0,0 +1,38 @@ +/* tslint:disable:no-console */ +/** + * Created by SUN-ASTERISK\dinh.van.hoang on 2/24/20 + */ + +import { DynamoDB } from 'aws-sdk'; +import { CreateTableInput } from 'aws-sdk/clients/dynamodb'; +import { ENVIRONMENTS } from '../config'; + +(async () => { + const dynamodb = new DynamoDB({ + endpoint: ENVIRONMENTS.DYNAMO_ENDPOINT, + region: ENVIRONMENTS.REGION, + }); + + const params: CreateTableInput = { + TableName: 'Movies', + KeySchema: [ + { AttributeName: 'year', KeyType: 'HASH' }, // Partition key + { AttributeName: 'title', KeyType: 'RANGE' }, // Sort key + ], + AttributeDefinitions: [ + { AttributeName: 'year', AttributeType: 'N' }, + { AttributeName: 'title', AttributeType: 'S' }, + ], + ProvisionedThroughput: { + ReadCapacityUnits: 10, + WriteCapacityUnits: 10, + }, + }; + + try { + const result = dynamodb.createTable(params).promise(); + console.log('Created table. Table description JSON:', JSON.stringify(result, null, 2)); + } catch (error) { + console.error('Unable to create table. Error JSON:', JSON.stringify(error, null, 2)); + } +})(); diff --git a/src/migrate/data/keep.me b/src/migrate/data/keep.me new file mode 100644 index 0000000..e69de29 diff --git a/src/migrate/load-simple-movies.ts b/src/migrate/load-simple-movies.ts new file mode 100644 index 0000000..26d65c8 --- /dev/null +++ b/src/migrate/load-simple-movies.ts @@ -0,0 +1,58 @@ +/* tslint:disable:no-console */ + +/** + * Created by SUN-ASTERISK\dinh.van.hoang on 2/24/20 + */ +import { DynamoDB } from 'aws-sdk'; +import { DocumentClient } from 'aws-sdk/clients/dynamodb'; +import fs from 'fs'; +import { ENVIRONMENTS } from '../config'; + +(async () => { + const docClient = new DynamoDB.DocumentClient({ + endpoint: ENVIRONMENTS.DYNAMO_ENDPOINT, + region: ENVIRONMENTS.REGION, + }); + + console.log('Importing movies into DynamoDB. Please wait.'); + + const allMovies: { + year: number; + title: string; + info: string; + }[] = JSON.parse(fs.readFileSync(`${__dirname}/data/moviedata.json`, 'utf8')); + + let promisesBatch: Promise[] = []; + for (const movie of allMovies) { + const params: DocumentClient.PutItemInput = { + TableName: 'Movies', + Item: { + 'year': movie.year, + 'title': movie.title, + 'info': movie.info, + }, + }; + + const promise = docClient.put(params).promise() + .then(_ => { + console.error('Done: ', movie.title); + }) + .catch(reason => { + // Skip error and continue + console.error('Error JSON:', JSON.stringify(reason, null, 2)); + }); + + promisesBatch.push(promise); + + if (promisesBatch.length === 100) { + await Promise.all(promisesBatch); + // Clear batch + promisesBatch = []; + } + } + + // Final block + await Promise.all(promisesBatch); + + console.error('Done!'); +})(); diff --git a/tslint.json b/tslint.json index 29649a3..2d05889 100644 --- a/tslint.json +++ b/tslint.json @@ -6,7 +6,14 @@ "jsRules": {}, "rules": { "quotemark": [true, "single"], - "import-spacing": true + "import-spacing": true, + "ordered-imports": true, + "trailing-comma": [ + true, + { + "multiline": "always" + } + ] }, "rulesDirectory": [] }