From 6f33f7a7494ea18e998ec9912f1f72d3e7dc2e19 Mon Sep 17 00:00:00 2001 From: Benny Chew Date: Sun, 28 Jan 2018 13:19:25 +1100 Subject: [PATCH] CI fixes - Add BOTO_CONFIG env to fix travis-ci, thanks @loomchild @MattSurabian - Switch dynalite to AWS DynamoDB Local - Remove nodejs requirement - Fix tests to work with DynamoDB Local - Update readme --- .travis.yml | 28 +++++++++++++++++----------- README.md | 8 ++++---- test/testTable/schema.json | 34 ++++++++++++++++++---------------- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1371a772..46e096c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,26 +1,32 @@ language: python +env: + - BOTO_CONFIG=/dev/null python: - "2.7" +before_install: + # install and run latest DynamoDB Local + - mkdir /tmp/dynamodb_local + - wget -O - https://s3-us-west-2.amazonaws.com/dynamodb-local/dynamodb_local_latest.tar.gz | tar xz --directory /tmp/dynamodb_local + - java -Djava.library.path=/tmp/dynamodb_local/DynamoDBLocal_lib -jar /tmp/dynamodb_local/DynamoDBLocal.jar -sharedDb -inMemory & install: - pip install -r requirements.txt - pip install flake8 - - nvm install 4 - - nvm use 4 - - npm install dynalite script: - - flake8 --ignore=E501 --exclude=node_modules - - dynalite --port 4567 & + - flake8 --ignore=E501 - mkdir dump && cp -a test/testTable dump - - python dynamodump.py -m restore -r local -s testTable -d testRestoredTable --host localhost --port 4567 --accessKey a --secretKey a - - python dynamodump.py -m backup -r local -s testRestoredTable --host localhost --port 4567 --accessKey a --secretKey a + # test basic restore and backup + - python dynamodump.py -m restore -r local -s testTable -d testRestoredTable --host localhost --port 8000 --accessKey a --secretKey a + - python dynamodump.py -m backup -r local -s testRestoredTable --host localhost --port 8000 --accessKey a --secretKey a - python test/test.py - - python dynamodump.py -m restore -r local -s "*" --host localhost --port 4567 --accessKey a --secretKey a + # test wildcard restore and backup + - python dynamodump.py -m restore -r local -s "*" --host localhost --port 8000 --accessKey a --secretKey a - rm -rf dump/test* - - python dynamodump.py -m backup -r local -s "*" --host localhost --port 4567 --accessKey a --secretKey a + - python dynamodump.py -m backup -r local -s "*" --host localhost --port 8000 --accessKey a --secretKey a - python test/test.py - - python dynamodump.py -m restore -r local -s "test*" --host localhost --port 4567 --accessKey a --secretKey a --prefixSeparator "" + # test prefixed wildcard restore and backup + - python dynamodump.py -m restore -r local -s "test*" --host localhost --port 8000 --accessKey a --secretKey a --prefixSeparator "" - rm -rf dump/test* - - python dynamodump.py -m backup -r local -s "test*" --host localhost --port 4567 --accessKey a --secretKey a --prefixSeparator "" + - python dynamodump.py -m backup -r local -s "test*" --host localhost --port 8000 --accessKey a --secretKey a --prefixSeparator "" - python test/test.py matrix: fast_finish: true diff --git a/README.md b/README.md index e2022ae9..3e168095 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Simple backup and restore script for Amazon DynamoDB using boto to work similarl Suitable for DynamoDB usages of smaller data volume which do not warrant the usage of AWS Data Pipeline for backup/restores/empty. -dynamodump supports local DynamoDB instances as well (tested with [dynalite](https://github.com/mhart/dynalite)). +dynamodump supports local DynamoDB instances as well (tested with [DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.html)). Usage ----- @@ -141,10 +141,10 @@ python2 dynamodump.py -a tar -b some_s3_bucket -m restore -r us-east-1 -p profil Local example ------------- -The following assume your local DynamoDB is running on localhost:4567 and is accessible via 'a' as access/secret keys. +The following assume your local DynamoDB is running on localhost:8000 and is accessible via 'a' as access/secret keys. ``` -python dynamodump.py -m backup -r local -s testTable --host localhost --port 4567 --accessKey a --secretKey a +python dynamodump.py -m backup -r local -s testTable --host localhost --port 8000 --accessKey a --secretKey a -python dynamodump.py -m restore -r local -s testTable --host localhost --port 4567 --accessKey a --secretKey a +python dynamodump.py -m restore -r local -s testTable --host localhost --port 8000 --accessKey a --secretKey a ``` Multiple table backup/restore as stated in the AWS examples are also available for local. diff --git a/test/testTable/schema.json b/test/testTable/schema.json index c9f47e98..4c01257b 100644 --- a/test/testTable/schema.json +++ b/test/testTable/schema.json @@ -1,28 +1,30 @@ { "Table": { - "TableArn": "arn:aws:dynamodb:us-east-1:000000000000:table/testTable", + "TableArn": "arn:aws:dynamodb:ddblocal:000000000000:table/testTable", "AttributeDefinitions": [ { - "AttributeName": "id", + "AttributeName": "id", "AttributeType": "N" } - ], + ], "ProvisionedThroughput": { - "WriteCapacityUnits": 25, - "NumberOfDecreasesToday": 1, - "ReadCapacityUnits": 1, - "LastDecreaseDateTime": 1471785062.495 - }, - "TableSizeBytes": 0, - "TableName": "testTable", - "TableStatus": "ACTIVE", + "NumberOfDecreasesToday": 0, + "WriteCapacityUnits": 25, + "LastIncreaseDateTime": 0.0, + "NumberOfDecreasesToday": 0, + "ReadCapacityUnits": 1, + "LastDecreaseDateTime": 0.0 + }, + "TableSizeBytes": 28, + "TableName": "testTable", + "TableStatus": "ACTIVE", "KeySchema": [ { - "KeyType": "HASH", + "KeyType": "HASH", "AttributeName": "id" } - ], - "ItemCount": 0, - "CreationDateTime": 1471785060.962 + ], + "ItemCount": 1, + "CreationDateTime": 1517103019.926 } -} +} \ No newline at end of file