Merge pull request #786 from LibraryOfCongress/update-db-scripts

one step closer to fully automating data dump / restore
LibraryOfCongress · Jan 31, 2019 · f94f811 · f94f811
2 parents 972e54a + cc71a3f
commit f94f811
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 37 deletions.
diff --git a/build_containers.sh b/build_containers.sh
@@ -5,8 +5,9 @@ set -eu -o pipefail
 BUILD_ALL=${BUILD_ALL:=0}
 BUILD_NUMBER=${BUILD_NUMBER:=1}
 TAG=${TAG:-test}
+PUBLISH_CONTAINERS=${PUBLISH_CONTAINERS:=1}
 
-# Get an unique venv folder to using *inside* workspace
+# Get an unique venv folder to use inside workspace
 VENV=".venv-${BUILD_NUMBER}"
 
 # Initialize new venv
@@ -18,40 +19,45 @@ source "${VENV}/bin/activate"
 pip3 install pipenv
 pipenv install --dev --deploy
 
-AWS_ACCOUNT_ID="$(aws sts get-caller-identity  --output=text --query "Account")"
-
 FULL_VERSION_NUMBER="$(python3 setup.py --version)"
-
 VERSION_NUMBER=$(echo "${FULL_VERSION_NUMBER}" | cut -d '+' -f 1)
 
-eval "$(aws ecr get-login --no-include-email --region us-east-1)"
+if [ $PUBLISH_CONTAINERS -eq 1 ]; then
+    AWS_ACCOUNT_ID="$(aws sts get-caller-identity  --output=text --query "Account")"
+    eval "$(aws ecr get-login --no-include-email --region us-east-1)"
+fi
 
 python3 setup.py build
 
 docker build -t concordia .
-docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
-docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
-docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
-docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
+
+if [ $PUBLISH_CONTAINERS -eq 1 ]; then
+    docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
+    docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
+    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
+    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
+fi
 
 if [ $BUILD_ALL -eq 1 ]; then
 
     docker build -t concordia/importer --file importer/Dockerfile .
-    docker tag concordia/importer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${VERSION_NUMBER}"
-    docker tag concordia/importer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${TAG}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${VERSION_NUMBER}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${TAG}"
-
     docker build -t concordia/celerybeat --file celerybeat/Dockerfile .
-    docker tag concordia/celerybeat:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${VERSION_NUMBER}"
-    docker tag concordia/celerybeat:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${TAG}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${VERSION_NUMBER}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${TAG}"
-
     docker build -t concordia/indexer --file indexer/Dockerfile .
-    docker tag concordia/indexer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${VERSION_NUMBER}"
-    docker tag concordia/indexer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${VERSION_NUMBER}"
-    docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}"
 
+    if [ $PUBLISH_CONTAINERS -eq 1 ]; then
+        docker tag concordia/importer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${VERSION_NUMBER}"
+        docker tag concordia/importer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${TAG}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${VERSION_NUMBER}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/importer:${TAG}"
+
+        docker tag concordia/celerybeat:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${VERSION_NUMBER}"
+        docker tag concordia/celerybeat:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${TAG}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${VERSION_NUMBER}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/celerybeat:${TAG}"
+
+        docker tag concordia/indexer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${VERSION_NUMBER}"
+        docker tag concordia/indexer:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${VERSION_NUMBER}"
+        docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}"
+    fi
 fi
diff --git a/db_scripts/dump.sh b/db_scripts/dump.sh
@@ -1,6 +1,29 @@
 #!/bin/bash
 
-# For AMI: sudo yum -y install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-ami201503-96-9.6-2.noarch.rpm
-#          sudo yum -y install postgresql96
+set -eu -o pipefail
 
-pg_dump -Fc --clean --create --no-owner --no-acl -U concordia -h $POSTGRESQL_HOST concordia -f concordia.dmp
+ENV_NAME=prod
+
+# aws cloudformation create-stack --region us-east-1 --stack-name $ENV_NAME-bastionhosts --template-url https://s3.amazonaws.com/crowd-deployment/infrastructure/bastion-hosts.yaml --parameters ParameterKey=EnvironmentName,ParameterValue=$ENV_NAME ParameterKey=KeyPairName,ParameterValue=rstorey@loc.gov --disable-rollback
+# aws cloudformation delete-stack --region us-east-1 --stack-name $ENV_NAME-bastionhosts
+
+if [ $ENV_NAME != "prod" ]; then
+    echo "This script should only be run in the production environment."
+    exit 1
+fi
+
+TODAY=$(date +%Y%m%d)
+POSTGRESQL_PW="$(aws secretsmanager get-secret-value --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret["SecretString"]);print(SecretString["password"])')"
+# TODO: look up the RDS endpoint for this environment
+POSTGRESQL_HOST=${POSTGRESQL_HOST:-localhost}
+DUMP_FILE=concordia.dmp
+
+echo "${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}" > ~/.pgpass
+chmod 600 ~/.pgpass
+
+pg_dump -Fc --clean --create --no-owner --no-acl -U concordia -h "${POSTGRESQL_HOST}" concordia -f "${DUMP_FILE}"
+
+if [ -s $DUMP_FILE ]; then
+    aws s3 cp "${DUMP_FILE}" "s3://crowd-deployment/database-dumps/concordia.${TODAY}.dmp"
+    aws s3 cp "${DUMP_FILE}" s3://crowd-deployment/database-dumps/concordia.latest.dmp
+fi
diff --git a/db_scripts/restore.sh b/db_scripts/restore.sh
@@ -1,16 +1,24 @@
-
 #!/bin/bash
 
-# For AMI: sudo yum -y install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-ami201503-96-9.6-2.noarch.rpm
-#          sudo yum -y install postgresql96
+set -eu -o pipefail
+
+ENV_NAME=dev
+
+# aws cloudformation create-stack --region us-east-1 --stack-name $ENV_NAME-bastionhosts --template-url https://s3.amazonaws.com/crowd-deployment/infrastructure/bastion-hosts.yaml --parameters ParameterKey=EnvironmentName,ParameterValue=$ENV_NAME ParameterKey=KeyPairName,ParameterValue=rstorey@loc.gov --disable-rollback
+# aws cloudformation delete-stack --region us-east-1 --stack-name $ENV_NAME-bastionhosts
+if [ $ENV_NAME = "prod" ]; then
+    echo "This script should not be run in the production environment."
+    exit 1
+fi
 
-# Before running the restore, you'll have to stop the ECS task to close any open connections.
-# Then, run the following to drop the database.
-# psql -U concordia -h $POSTGRESQL_HOST
-# \c postgres
-# drop database concordia;
+POSTGRESQL_PW="$(aws secretsmanager get-secret-value --secret-id crowd/${ENV_NAME}/DB/MasterUserPassword | python3 -c 'import json,sys;Secret=json.load(sys.stdin);SecretString=json.loads(Secret["SecretString"]);print(SecretString["password"])')"
+# TODO: look up the RDS endpoint for this environment
+POSTGRESQL_HOST=${POSTGRESQL_HOST:-localhost}
+DUMP_FILE=/concordia.dmp
 
-export POSTGRESQL_HOST=${POSTGRESQL_HOST:-localhost}
-export DUMP_FILE=concordia.dmp
+echo "${POSTGRESQL_HOST}:5432:*:concordia:${POSTGRESQL_PW}" > ~/.pgpass
+chmod 600 ~/.pgpass
 
-pg_restore --create --clean -U concordia -h "${POSTGRESQL_HOST}" -Fc --dbname=postgres --no-owner --no-acl $DUMP_FILE
+psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "select pg_terminate_backend(pid) from pg_stat_activity where datname='concordia';"
+psql -U concordia -h "$POSTGRESQL_HOST" -d postgres -c "drop database concordia;"
+pg_restore --create --clean -U concordia -h "${POSTGRESQL_HOST}" -Fc --dbname=postgres --no-owner --no-acl "${DUMP_FILE}"