From ec0ebd12f4b77152afe46ff07d8696d02577fc56 Mon Sep 17 00:00:00 2001 From: Andrew Purser Date: Sun, 6 Dec 2020 18:53:09 +0700 Subject: [PATCH 1/2] add additional steps to test setup instructions --- CONTRIBUTING.md | 37 +++++++++++++++++-- cloudformation/databases.yaml | 5 +++ cloudformation/delete-base.sh | 6 +++ cloudformation/delete-databases.sh | 6 +++ .../security-group-databases-add-local-ip.sh | 16 ++++++++ .../security-group-databases-check.sh | 9 +++++ 6 files changed, 76 insertions(+), 3 deletions(-) create mode 100755 cloudformation/delete-base.sh create mode 100755 cloudformation/delete-databases.sh create mode 100755 cloudformation/security-group-databases-add-local-ip.sh create mode 100755 cloudformation/security-group-databases-check.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fad7a2254..027b9d901 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -100,7 +100,7 @@ You can choose from three different environments to test your fixes/changes, bas * To run a specific test function: -``pytest tests/test_moto::test_get_bucket_region_succeed`` +``pytest tests/test_moto.py::test_get_bucket_region_succeed`` * To run all mocked test functions (Using 8 parallel processes): @@ -121,6 +121,10 @@ You can choose from three different environments to test your fixes/changes, bas ``pip install -r requirements-dev.txt`` +* [OPTIONAL] Set AWS_DEFAULT_REGION to define the region the Data Lake Test envrioment will deploy into. You may want to choose a region which you don't currently use: + +``export AWS_DEFAULT_REGION=ap-northeast-1`` + * Go to the ``cloudformation`` directory ``cd cloudformation`` @@ -145,10 +149,16 @@ You can choose from three different environments to test your fixes/changes, bas ``pytest -n 8 tests/test_athena*`` +* [OPTIONAL] To remove the base test environment cloud formation stack post testing: + +``./cloudformation/delete-base.sh`` + ### Full test environment **DISCLAIMER**: Make sure you know what you are doing. These steps will charge some services on your AWS account and require a minimum security skill to keep your environment safe. +**DISCLAIMER**: This environment contains Aurora MySQL, Aurora PostgreSQL and Redshift (single-node) clusters which will incur cost while running. + * Pick up a Linux or MacOS. * Install Python 3.6, 3.7 and 3.8 * Fork the AWS Data Wrangler repository and clone that into your development environment @@ -160,16 +170,27 @@ You can choose from three different environments to test your fixes/changes, bas ``pip install -r requirements-dev.txt`` +* [OPTIONAL] Set AWS_DEFAULT_REGION to define the region the Full Test envrioment will deploy into. You may want to choose a region which you don't currently use: + +``export AWS_DEFAULT_REGION=ap-northeast-1`` + * Go to the ``cloudformation`` directory ``cd cloudformation`` -* Deploy the Cloudformation templates `base.yaml` and `databases.yaml` +* Deploy the Cloudformation templates `base.yaml` and `databases.yaml`. This step could take about 15 minutes to deploy. ``./deploy-base.sh`` ``./deploy-databases.sh`` * Go to the `EC2 -> SecurityGroups` console, open the `aws-data-wrangler-*` security group and configure to accept your IP from any TCP port. + - Alternatively run: + + ``./security-group-databases-add-local-ip.sh`` + + - Check local IP was applied: + + ``./security-group-databases-check.sh`` ``P.S Make sure that your security group will not be open to the World! Configure your security group to only give access for your IP.`` @@ -185,12 +206,22 @@ You can choose from three different environments to test your fixes/changes, bas * To run a specific test function: -``pytest tests/test_athena_parquet.py::test_parquet_catalog`` +``pytest tests/test_db.py::test_sql`` + +* To run all database test functions (Using 8 parallel processes): + +``pytest -n 8 tests/test_db.py`` * To run all data lake test functions for all python versions (Only if Amazon QuickSight is activated): ``./test.sh`` +* [OPTIONAL] To remove the base test environment cloud formation stack post testing: + +``./cloudformation/delete-base.sh`` + +``./cloudformation/delete-databases.sh`` + ## Recommended Visual Studio Code Recommended settings ```json diff --git a/cloudformation/databases.yaml b/cloudformation/databases.yaml index 80112dc3c..0166bf8af 100644 --- a/cloudformation/databases.yaml +++ b/cloudformation/databases.yaml @@ -408,3 +408,8 @@ Outputs: - AuroraInstanceMysql - Endpoint.Address Description: Mysql Address + DatabaseSecurityGroupId: + Value: + Fn::GetAtt: + - DatabaseSecurityGroup + - GroupId diff --git a/cloudformation/delete-base.sh b/cloudformation/delete-base.sh new file mode 100755 index 000000000..06b79ed34 --- /dev/null +++ b/cloudformation/delete-base.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +# Deploying +aws cloudformation delete-stack \ + --stack-name aws-data-wrangler-base diff --git a/cloudformation/delete-databases.sh b/cloudformation/delete-databases.sh new file mode 100755 index 000000000..dfbbb3806 --- /dev/null +++ b/cloudformation/delete-databases.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -e + +# Deleting +aws cloudformation delete-stack \ + --stack-name aws-data-wrangler-databases diff --git a/cloudformation/security-group-databases-add-local-ip.sh b/cloudformation/security-group-databases-add-local-ip.sh new file mode 100755 index 000000000..4ceb74528 --- /dev/null +++ b/cloudformation/security-group-databases-add-local-ip.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +# Get my current IP address +LOCALIP=`host myip.opendns.com resolver1.opendns.com | grep myip | awk '{print $4}'` + +# Get security group ID +SGID=`aws cloudformation describe-stacks --stack-name aws-data-wrangler-databases --query "Stacks[0].Outputs[?OutputKey=='DatabaseSecurityGroupId'].OutputValue" --output text` + +# Update Security Group with local ip +aws ec2 authorize-security-group-ingress \ + --group-id ${SGID} \ + --protocol all \ + --port -1 \ + --cidr ${LOCALIP}/32 \ + --output text diff --git a/cloudformation/security-group-databases-check.sh b/cloudformation/security-group-databases-check.sh new file mode 100755 index 000000000..e3d8ea5b2 --- /dev/null +++ b/cloudformation/security-group-databases-check.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# Get security group ID +SGID=`aws cloudformation describe-stacks --stack-name aws-data-wrangler-databases --query "Stacks[0].Outputs[?OutputKey=='DatabaseSecurityGroupId'].OutputValue" --output text` + +# Check to see current setting +aws ec2 describe-security-groups \ + --group-id ${SGID} \ \ No newline at end of file From 352c105a2091aa45ff5ed39b31ae34c2d8261c34 Mon Sep 17 00:00:00 2001 From: Andrew Purser Date: Sun, 6 Dec 2020 19:01:07 +0700 Subject: [PATCH 2/2] add additional steps to test setup instructions --- cloudformation/security-group-databases-add-local-ip.sh | 3 +-- cloudformation/security-group-databases-check.sh | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cloudformation/security-group-databases-add-local-ip.sh b/cloudformation/security-group-databases-add-local-ip.sh index 4ceb74528..4cad07f0f 100755 --- a/cloudformation/security-group-databases-add-local-ip.sh +++ b/cloudformation/security-group-databases-add-local-ip.sh @@ -12,5 +12,4 @@ aws ec2 authorize-security-group-ingress \ --group-id ${SGID} \ --protocol all \ --port -1 \ - --cidr ${LOCALIP}/32 \ - --output text + --cidr ${LOCALIP}/32 diff --git a/cloudformation/security-group-databases-check.sh b/cloudformation/security-group-databases-check.sh index e3d8ea5b2..a3f0a6a5c 100755 --- a/cloudformation/security-group-databases-check.sh +++ b/cloudformation/security-group-databases-check.sh @@ -5,5 +5,4 @@ set -e SGID=`aws cloudformation describe-stacks --stack-name aws-data-wrangler-databases --query "Stacks[0].Outputs[?OutputKey=='DatabaseSecurityGroupId'].OutputValue" --output text` # Check to see current setting -aws ec2 describe-security-groups \ - --group-id ${SGID} \ \ No newline at end of file +aws ec2 describe-security-groups --group-id ${SGID}