From 158fa032dba5923b9a40dd22b9832b92d13f4647 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Thu, 4 Jul 2019 16:08:56 -0230 Subject: [PATCH 01/11] refactor: cleanup and optimize --- .gitignore | 1 + Dockerfile | 42 ++++++++---- README.md | 16 +++++ action.sh | 158 +++++++++++++++++++++++++++++---------------- build_push.sh | 23 +++---- docker-compose.yml | 28 ++++++++ 6 files changed, 186 insertions(+), 82 deletions(-) create mode 100644 .gitignore create mode 100644 docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e0cc99 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +docker-compose.override.yml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 85fc3ff..519b103 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,35 @@ -ARG pg_alpine_branch -FROM alpine:${pg_alpine_branch} +ARG PG_ALPINE_BRANCH +FROM alpine:${PG_ALPINE_BRANCH} -ARG pg_alpine_branch -ARG pg_version +ARG PG_ALPINE_BRANCH +ARG PG_VERSION -# python for aws-cli, for s3 downloading -RUN apk --no-cache add python py-pip && \ +#-------------------------------------------------------------------------------- +# Install dependencies +#-------------------------------------------------------------------------------- +# "postgresql" is required for "pg_restore" +# "python" is required for "aws-cli" +#-------------------------------------------------------------------------------- +RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${PG_ALPINE_BRANCH}/main" >> /etc/apk/repositories + +RUN apk --no-cache add dumb-init postgresql=${PG_VERSION} python py-pip && \ pip install awscli && \ apk --purge -v del py-pip -# postgresql for pg_restore -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${pg_alpine_branch}/main" >> /etc/apk/repositories -RUN apk --no-cache add postgresql=${pg_version} - -COPY action.sh / -RUN chmod +x action.sh +#-------------------------------------------------------------------------------- +# Set script permissions and create required directories +#-------------------------------------------------------------------------------- +COPY aws-mfa.sh action.sh / +RUN chmod +x action.sh && chmod +x aws-mfa.sh +RUN mkdir -p /cache && mkdir -p /root/.aws -RUN mkdir -p /cache +#-------------------------------------------------------------------------------- +# Use the `dumb-init` init system (PID 1) for process handling +#-------------------------------------------------------------------------------- +ENTRYPOINT ["/usr/bin/dumb-init", "--"] -CMD echo "${CRON_MINUTE:-$(shuf -i 0-59 -n1)} ${CRON_HOUR:-*} * * * /action.sh" > /var/spool/cron/crontabs/root && crond -d 8 -f +#-------------------------------------------------------------------------------- +# Configure and apply a cronjob +#-------------------------------------------------------------------------------- +CMD echo "${CRON_MINUTE:-$(shuf -i 0-59 -n1)} ${CRON_HOUR:-*} * * * /action.sh" \ +> /var/spool/cron/crontabs/root && crond -d 8 -f diff --git a/README.md b/README.md index f1bc80f..2240867 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,22 @@ To execute arbitrary psql / SQL commands before or after the internal _pg_restor -e PRE_RESTORE_PSQL="____" \ # "CREATE EXTENSION postgis; CREATE EXTENSION pg_trgm;" ``` +### AWS multi-factor authentication + +If an AWS MFA code isn't passed to `action.sh` as an argument it will be requested. + +Authentication via AWS MFA can be enabled by setting the following environment variable. + +``` +-e AWS_MFA_DEVICE_ARN= +``` + +In order for the temporary AWS MFA credentials to persist until they expire a volume must be mounted. + +``` +-v /tmp/.aws/:/root/.aws/ +``` + ***Note**: the usual cron tricks apply to the hour and minute env values. For instance setting `CRON_HOUR` to `*/4` and `CRON_MINUTE` to `0`, will trigger once every 4 hours.* Creating database dumps can be accomplished with the `bluedrop360/postgres-dump-to-s3` repo. diff --git a/action.sh b/action.sh index 337272c..140efef 100755 --- a/action.sh +++ b/action.sh @@ -1,72 +1,116 @@ -#!/bin/sh +#!/usr/bin/env sh -echo "postgres restore from s3 - looking for dump in cache and on s3 at s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}" -if [ -n "${DUMP_OBJECT}" ]; then - object=${DUMP_OBJECT} - dumpFile=$(echo ${DUMP_OBJECT} | sed 's/.*\///') +################################################################ +# Variable definitions +################################################################ +# shellcheck disable=SC2001 +DB_NAME=$(echo "${DATABASE_URL}" | sed "s|.*/\([^/]*\)\$|\\1|") + +# shellcheck disable=SC2001 +DB_ROOT_URL=$(echo "${DATABASE_URL}" | sed "s|/[^/]*\$|/template1|") + +DROP_RESULT=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '${DB_NAME}'; \ +DROP DATABASE ${DB_NAME};" | psql "${DB_ROOT_URL}" 2>&1) + +################################################################ +# Locate the dump file in the cache or from AWS S3 +################################################################ +printf '%b\n' "\n> Searching for a dump file in the local cache..." + +if [ -n "$DUMP_OBJECT" ]; then + OBJECT=${DUMP_OBJECT} + DUMP_FILE=$(echo "${DUMP_OBJECT}" | sed 's/.*\///') else - if [ -n "${DUMP_OBJECT_DATE}" ]; then - dateFilter=${DUMP_OBJECT_DATE} - else - dateFilter=$(date +"%Y-%m-%dT%H:%M") - fi - # broaden filter until a match is found that is also less than dateFilter - filter=$dateFilter - while true; do - echo "postgres restore from s3 - using filter $filter" - if [ -f "/cache/$filter.dump" ]; then - # file exists in the cache, stop looking remotely - object=$filter - dumpFile=$filter.dump - break; - fi - dumpFile=$(aws --region ${AWS_REGION} s3 ls s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${filter} | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1) - if [ -n "$dumpFile" ]; then - object=${DUMP_OBJECT_PREFIX}$dumpFile - # found an object, success - break; - fi - if [ -z "$filter" ]; then - # got to an empty filter and still nothing found - object="" - break; + if [ -n "$DUMP_OBJECT_DATE" ]; then + FILTER=${DUMP_OBJECT_DATE} + else + FILTER=$(date +"%Y-%m-%dT%H:%M") fi - filter="${filter%?}" - done + + # Broaden filter until a match is found that is also less than FILTER + while true; do + printf '%b\n' " Trying filter: ${FILTER}" + + # File exists in the cache, stop looking remotely + if [ -f "/cache/$FILTER.dump" ]; then + OBJECT=${FILTER} + DUMP_FILE=${FILTER}.dump + break; + fi + + DUMP_FILE=$(aws --region "${AWS_REGION}" s3 ls "s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${FILTER}" | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1) + + # Found an object, success + if [ -n "${DUMP_FILE}" ]; then + OBJECT=${DUMP_OBJECT_PREFIX}${DUMP_FILE} + break; + fi + + # Got to an empty filter and still nothing found + if [ -z "$FILTER" ]; then + OBJECT="" + break; + fi + + FILTER="${FILTER%?}" + done fi -if [ -z "$object" ]; then - echo "postgres restore from s3 - dump file not found on s3" - exit 1 + +if [ -z "$OBJECT" ]; then + printf '%b\n' "> Dump file not found in AWS S3 bucket" + exit 1 fi -if [ -f "/cache/$dumpFile" ]; then - echo "postgres restore from s3 - using cached $dumpFile" + +if [ -f "/cache/${DUMP_FILE}" ]; then + printf '%b\n' " Using cached dump: \"${DUMP_FILE}\"" else - echo "postgres restore from s3 - downloading dump from s3 - $object" - aws --region ${AWS_REGION} s3 cp s3://${AWS_BUCKET}/$object /cache/$dumpFile + printf '%b\n' " Not found: Attempting to download the dump from an AWS S3 bucket" + + # Download the dump + printf '%b\n' "\n> Downloading the latest dump from: \"s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}\"" + aws --region "${AWS_REGION}" s3 cp "s3://${AWS_BUCKET}/${OBJECT}" "/cache/${DUMP_FILE}" || exit 1 fi -echo "postgres restore from s3 - dropping old database" -dbName=$(echo $DATABASE_URL | sed "s|.*/\([^/]*\)\$|\\1|") -dbRootUrl=$(echo $DATABASE_URL | sed "s|/[^/]*\$|/template1|") -dropResult=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$dbName'; \ - DROP DATABASE $dbName;" | psql $dbRootUrl 2>&1) -if echo $dropResult | grep "other session using the database" -> /dev/null; then - echo "RESTORE FAILED - another database session is preventing drop of database $dbName" - exit 1 + +################################################################ +# Drop the target database +################################################################ +printf '%b\n' '\n> Dropping the target database...' +printf '%b\n' " DROP DATABASE ${DB_NAME};" + +if echo "${DROP_RESULT}" | grep "other session using the database" >/dev/null 2>&1; then + echo "RESTORE FAILED - another database session is preventing drop of database ${DB_NAME}" + exit 1 fi -createResult=$(echo "CREATE DATABASE $dbName;" | psql $dbRootUrl 2>&1) -echo "postgres restore from s3 - filling target database with dump" + +################################################################ +# Restore the target database +################################################################ +printf '%b\n' '\n> Restoring the target database...' +printf '%b\n' " CREATE DATABASE ${DB_NAME};\n REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC;\n ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};" + +printf '%s' \ +"CREATE DATABASE ${DB_NAME}; REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC; ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};" | \ +psql "${DB_ROOT_URL}" >/dev/null 2>&1 + +printf '%b\n' "\n> Rebuilding the target database..." + if [ -n "$PRE_RESTORE_PSQL" ]; then - echo "postgres restore from s3 - executing pre restore psql" - printf %s "$PRE_RESTORE_PSQL" | psql $DATABASE_URL + printf '%b\n' "> Executing pre-restore psql" + printf '%b\n' "${PRE_RESTORE_PSQL}" | psql "${DATABASE_URL}" fi + if [ -n "$SCHEMA" ]; then - echo "postgres restore from s3 - schema - $SCHEMA" - pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d $DATABASE_URL /cache/$dumpFile + printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d /cache/${DUMP_FILE}" + pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --schema "$SCHEMA" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}" else - pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d $DATABASE_URL /cache/$dumpFile + printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d /cache/${DUMP_FILE}" + pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}" fi + if [ -n "$POST_RESTORE_PSQL" ]; then - echo "postgres restore from s3 - executing post restore psql" - printf %s "$POST_RESTORE_PSQL" | psql $DATABASE_URL + printf '%b\n' "> Executing post-restore psql" + printf '%s' "${POST_RESTORE_PSQL}" | psql "${DATABASE_URL}" fi -echo "postgres restore from s3 - complete - $object" + +echo "" +echo "COMPLETE: ${OBJECT}" diff --git a/build_push.sh b/build_push.sh index 3fe8dba..ea610c2 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,20 +1,21 @@ -#! /bin/sh +#!/usr/bin/env sh -builds=$(echo ' +BUILDS=$(echo ' 9.6.10 9.6.10-r0 3.6 9.6 9.6.10-r0 3.6 9 9.6.10-r0 3.6 latest 9.6.10-r0 3.6 ' | grep -v '^#' | tr -s ' ') +# shellcheck disable=SC2039 IFS=$'\n' -for build in $builds; do - tag=$(echo $build | cut -d ' ' -f 1 ) - pgVersion=$(echo $build | cut -d ' ' -f 2) - pgAlpineBranch=$(echo $build | cut -d ' ' -f 3) - - echo docker build --tag bluedrop360/postgres-restore-from-s3:$tag --build-arg pg_version=$pgVersion --build-arg pg_alpine_branch=$pgAlpineBranch . - eval docker build --tag bluedrop360/postgres-restore-from-s3:$tag --build-arg pg_version=$pgVersion --build-arg pg_alpine_branch=$pgAlpineBranch . - echo docker push bluedrop360/postgres-restore-from-s3:$tag - eval docker push bluedrop360/postgres-restore-from-s3:$tag +for BUILD in $BUILDS; do + TAG=$(echo "${BUILD}" | cut -d ' ' -f 1 ) + PG_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 2) + PG_ALPINE_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 3) + + echo docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" . + eval docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" . + echo docker push bluedrop360/postgres-restore-from-s3:"${TAG}" + eval docker push bluedrop360/postgres-restore-from-s3:"${TAG}" done diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6f753c2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +######################################################## +# Docker Compose: https://docs.docker.com/compose +######################################################## +# This file is for local testing +# - Copy contents to `docker-compose.override.yml` +# - Update the volume and environment variable values +# - BUILD: docker-compose build +# - BUILD AND RUN: docker-compose up --build +######################################################## +version: '3' + +services: + postgres-restore-from-s3: + image: postgres-restore-from-s3:9.6.10 + network_mode: 'host' + build: + context: ./ + dockerfile: ./Dockerfile + args: + PG_ALPINE_BRANCH: '3.6' + PG_VERSION: '9.6.10-r0' + environment: + AWS_BUCKET: + AWS_REGION: + DATABASE_URL: postgres://:@:/ + DUMP_OBJECT_PREFIX: /postgres/ + volumes: + - /build/postgres/dump:/cache:rw From 386b9f29078fdba6e11cddd117a63a4876c14fc5 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Sat, 13 Jul 2019 15:16:42 -0230 Subject: [PATCH 02/11] system: add lint script --- lint.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lint.sh diff --git a/lint.sh b/lint.sh new file mode 100644 index 0000000..be6fb52 --- /dev/null +++ b/lint.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env sh + +ALLOW_EXTERNAL_SOURCE="${ALLOW_EXTERNAL_SOURCE:-}" + +if [ "${1}" = "--yml" ]; then + printf '\n%s\n' "Checking '.travis.yml'..." + travis lint ./.travis.yml +fi + +if [ "${1}" = "--sh" ]; then + printf '\n%s\n' "Checking shell scripts..." + + SHELLCHECK_OPTS="" + + RUN_SHELLCHECK="shellcheck ${ALLOW_EXTERNAL_SOURCE} ${SHELLCHECK_OPTS} {} +" + eval "find ./*.sh -type f -exec ${RUN_SHELLCHECK}" +fi From 5f4495aeaca1fcbc199fb5801745ee838ec42451 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Sat, 13 Jul 2019 15:18:27 -0230 Subject: [PATCH 03/11] ci: add 'travis' script --- .travis.yml | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..a42996d --- /dev/null +++ b/.travis.yml @@ -0,0 +1,32 @@ +language: node_js + +notifications: + email: false + +sudo: false + +node_js: + - "10" + +dist: trusty + +before_install: + - sudo apt-add-repository "deb http://archive.ubuntu.com/ubuntu trusty-backports main restricted universe multiverse" + - sudo apt-get -qq update + - sudo apt-get install xz-utils + - export scversion="stable" + - wget "https://storage.googleapis.com/shellcheck/shellcheck-${scversion}.linux.x86_64.tar.xz" + - tar --xz -xvf shellcheck-"${scversion}".linux.x86_64.tar.xz + - sudo cp shellcheck-"${scversion}"/shellcheck /usr/bin/ + - shellcheck --version + - export ALLOW_EXTERNAL_SOURCE='-x ' + +env: + - CXX=g++-4.8 + +branches: + only: + - master + +script: + - ./lint.sh --sh From 572157597458a75c13e5ae97b3b36ebb632a9e23 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Sat, 13 Jul 2019 15:46:11 -0230 Subject: [PATCH 04/11] refactor: use correct letter case for arguments and variables --- Dockerfile | 12 ++++---- action.sh | 68 +++++++++++++++++++++++----------------------- build_push.sh | 18 ++++++------ docker-compose.yml | 4 +-- 4 files changed, 51 insertions(+), 51 deletions(-) diff --git a/Dockerfile b/Dockerfile index 519b103..e5e022e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -ARG PG_ALPINE_BRANCH -FROM alpine:${PG_ALPINE_BRANCH} +ARG pg_alpine_branch +FROM alpine:${pg_alpine_branch} -ARG PG_ALPINE_BRANCH -ARG PG_VERSION +ARG pg_alpine_branch +ARG pg_version #-------------------------------------------------------------------------------- # Install dependencies @@ -10,9 +10,9 @@ ARG PG_VERSION # "postgresql" is required for "pg_restore" # "python" is required for "aws-cli" #-------------------------------------------------------------------------------- -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${PG_ALPINE_BRANCH}/main" >> /etc/apk/repositories +RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${pg_alpine_branch}/main" >> /etc/apk/repositories -RUN apk --no-cache add dumb-init postgresql=${PG_VERSION} python py-pip && \ +RUN apk --no-cache add dumb-init postgresql=${pg_version} python py-pip && \ pip install awscli && \ apk --purge -v del py-pip diff --git a/action.sh b/action.sh index 140efef..b6bfbfe 100755 --- a/action.sh +++ b/action.sh @@ -4,13 +4,13 @@ # Variable definitions ################################################################ # shellcheck disable=SC2001 -DB_NAME=$(echo "${DATABASE_URL}" | sed "s|.*/\([^/]*\)\$|\\1|") +db_name=$(echo "${DATABASE_URL}" | sed "s|.*/\([^/]*\)\$|\\1|") # shellcheck disable=SC2001 -DB_ROOT_URL=$(echo "${DATABASE_URL}" | sed "s|/[^/]*\$|/template1|") +db_root_url=$(echo "${DATABASE_URL}" | sed "s|/[^/]*\$|/template1|") -DROP_RESULT=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '${DB_NAME}'; \ -DROP DATABASE ${DB_NAME};" | psql "${DB_ROOT_URL}" 2>&1) +drop_result=$(echo "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '${db_name}'; \ +DROP DATABASE ${db_name};" | psql "${db_root_url}" 2>&1) ################################################################ # Locate the dump file in the cache or from AWS S3 @@ -18,67 +18,67 @@ DROP DATABASE ${DB_NAME};" | psql "${DB_ROOT_URL}" 2>&1) printf '%b\n' "\n> Searching for a dump file in the local cache..." if [ -n "$DUMP_OBJECT" ]; then - OBJECT=${DUMP_OBJECT} - DUMP_FILE=$(echo "${DUMP_OBJECT}" | sed 's/.*\///') + object=${DUMP_OBJECT} + dump_file=$(echo "${DUMP_OBJECT}" | sed 's/.*\///') else if [ -n "$DUMP_OBJECT_DATE" ]; then - FILTER=${DUMP_OBJECT_DATE} + filter=${DUMP_OBJECT_DATE} else - FILTER=$(date +"%Y-%m-%dT%H:%M") + filter=$(date +"%Y-%m-%dT%H:%M") fi - # Broaden filter until a match is found that is also less than FILTER + # Broaden filter until a match is found that is also less than filter while true; do - printf '%b\n' " Trying filter: ${FILTER}" + printf '%b\n' " Trying filter: ${filter}" # File exists in the cache, stop looking remotely - if [ -f "/cache/$FILTER.dump" ]; then - OBJECT=${FILTER} - DUMP_FILE=${FILTER}.dump + if [ -f "/cache/$filter.dump" ]; then + object=${filter} + dump_file=${filter}.dump break; fi - DUMP_FILE=$(aws --region "${AWS_REGION}" s3 ls "s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${FILTER}" | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1) + dump_file=$(aws --region "${AWS_REGION}" s3 ls "s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}${filter}" | sed "s/.* //" | grep '^[0-9:T\-]\{16\}\.dump$' | sort | tail -n 1) # Found an object, success - if [ -n "${DUMP_FILE}" ]; then - OBJECT=${DUMP_OBJECT_PREFIX}${DUMP_FILE} + if [ -n "${dump_file}" ]; then + object=${DUMP_OBJECT_PREFIX}${dump_file} break; fi # Got to an empty filter and still nothing found - if [ -z "$FILTER" ]; then - OBJECT="" + if [ -z "$filter" ]; then + object="" break; fi - FILTER="${FILTER%?}" + filter="${filter%?}" done fi -if [ -z "$OBJECT" ]; then +if [ -z "$object" ]; then printf '%b\n' "> Dump file not found in AWS S3 bucket" exit 1 fi -if [ -f "/cache/${DUMP_FILE}" ]; then - printf '%b\n' " Using cached dump: \"${DUMP_FILE}\"" +if [ -f "/cache/${dump_file}" ]; then + printf '%b\n' " Using cached dump: \"${dump_file}\"" else printf '%b\n' " Not found: Attempting to download the dump from an AWS S3 bucket" # Download the dump printf '%b\n' "\n> Downloading the latest dump from: \"s3://${AWS_BUCKET}/${DUMP_OBJECT_PREFIX}\"" - aws --region "${AWS_REGION}" s3 cp "s3://${AWS_BUCKET}/${OBJECT}" "/cache/${DUMP_FILE}" || exit 1 + aws --region "${AWS_REGION}" s3 cp "s3://${AWS_BUCKET}/${object}" "/cache/${dump_file}" || exit 1 fi ################################################################ # Drop the target database ################################################################ printf '%b\n' '\n> Dropping the target database...' -printf '%b\n' " DROP DATABASE ${DB_NAME};" +printf '%b\n' " DROP DATABASE ${db_name};" -if echo "${DROP_RESULT}" | grep "other session using the database" >/dev/null 2>&1; then - echo "RESTORE FAILED - another database session is preventing drop of database ${DB_NAME}" +if echo "${drop_result}" | grep "other session using the database" >/dev/null 2>&1; then + echo "RESTORE FAILED - another database session is preventing drop of database ${db_name}" exit 1 fi @@ -86,11 +86,11 @@ fi # Restore the target database ################################################################ printf '%b\n' '\n> Restoring the target database...' -printf '%b\n' " CREATE DATABASE ${DB_NAME};\n REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC;\n ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};" +printf '%b\n' " CREATE DATABASE ${db_name};\n REVOKE connect ON DATABASE ${db_name} FROM PUBLIC;\n ALTER DATABASE ${db_name} OWNER TO ${db_name};" printf '%s' \ -"CREATE DATABASE ${DB_NAME}; REVOKE connect ON DATABASE ${DB_NAME} FROM PUBLIC; ALTER DATABASE ${DB_NAME} OWNER TO ${DB_NAME};" | \ -psql "${DB_ROOT_URL}" >/dev/null 2>&1 +"CREATE DATABASE ${db_name}; REVOKE connect ON DATABASE ${db_name} FROM PUBLIC; ALTER DATABASE ${db_name} OWNER TO ${db_name};" | \ +psql "${db_root_url}" >/dev/null 2>&1 printf '%b\n' "\n> Rebuilding the target database..." @@ -100,11 +100,11 @@ if [ -n "$PRE_RESTORE_PSQL" ]; then fi if [ -n "$SCHEMA" ]; then - printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d /cache/${DUMP_FILE}" - pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --schema "$SCHEMA" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}" + printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d /cache/${dump_file}" + pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --schema "$SCHEMA" --no-owner -d "${DATABASE_URL}" "/cache/${dump_file}" else - printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d /cache/${DUMP_FILE}" - pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --no-owner -d "${DATABASE_URL}" "/cache/${DUMP_FILE}" + printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d /cache/${dump_file}" + pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --no-owner -d "${DATABASE_URL}" "/cache/${dump_file}" fi if [ -n "$POST_RESTORE_PSQL" ]; then @@ -113,4 +113,4 @@ if [ -n "$POST_RESTORE_PSQL" ]; then fi echo "" -echo "COMPLETE: ${OBJECT}" +echo "COMPLETE: ${object}" diff --git a/build_push.sh b/build_push.sh index ea610c2..cbdd0ba 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,6 +1,6 @@ #!/usr/bin/env sh -BUILDS=$(echo ' +builds=$(echo ' 9.6.10 9.6.10-r0 3.6 9.6 9.6.10-r0 3.6 9 9.6.10-r0 3.6 @@ -9,13 +9,13 @@ latest 9.6.10-r0 3.6 # shellcheck disable=SC2039 IFS=$'\n' -for BUILD in $BUILDS; do - TAG=$(echo "${BUILD}" | cut -d ' ' -f 1 ) - PG_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 2) - PG_ALPINE_VERSION=$(echo "${BUILD}" | cut -d ' ' -f 3) +for build in $builds; do + TAG=$(echo "${build}" | cut -d ' ' -f 1 ) + pg_version=$(echo "${build}" | cut -d ' ' -f 2) + pg_alpine_version=$(echo "${build}" | cut -d ' ' -f 3) - echo docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" . - eval docker build --tag bluedrop360/postgres-restore-from-s3:"${TAG}" --build-arg pg_version="${PG_VERSION}" --build-arg pg_alpine_branch="${PG_ALPINE_VERSION}" . - echo docker push bluedrop360/postgres-restore-from-s3:"${TAG}" - eval docker push bluedrop360/postgres-restore-from-s3:"${TAG}" + echo docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_version}" . + eval docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_version}" . + echo docker push bluedrop360/postgres-restore-from-s3:"${tag}" + eval docker push bluedrop360/postgres-restore-from-s3:"${tag}" done diff --git a/docker-compose.yml b/docker-compose.yml index 6f753c2..70c2799 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,8 @@ services: context: ./ dockerfile: ./Dockerfile args: - PG_ALPINE_BRANCH: '3.6' - PG_VERSION: '9.6.10-r0' + pg_alpine_branch: '3.6' + pg_version: '9.6.10-r0' environment: AWS_BUCKET: AWS_REGION: From 3a6ae3f0aca7f2996c70ecc44088cd030fc8c6a5 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Sun, 14 Jul 2019 23:57:51 -0230 Subject: [PATCH 05/11] doc: remove irrelevant section --- README.md | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/README.md b/README.md index 2240867..f1bc80f 100644 --- a/README.md +++ b/README.md @@ -70,22 +70,6 @@ To execute arbitrary psql / SQL commands before or after the internal _pg_restor -e PRE_RESTORE_PSQL="____" \ # "CREATE EXTENSION postgis; CREATE EXTENSION pg_trgm;" ``` -### AWS multi-factor authentication - -If an AWS MFA code isn't passed to `action.sh` as an argument it will be requested. - -Authentication via AWS MFA can be enabled by setting the following environment variable. - -``` --e AWS_MFA_DEVICE_ARN= -``` - -In order for the temporary AWS MFA credentials to persist until they expire a volume must be mounted. - -``` --v /tmp/.aws/:/root/.aws/ -``` - ***Note**: the usual cron tricks apply to the hour and minute env values. For instance setting `CRON_HOUR` to `*/4` and `CRON_MINUTE` to `0`, will trigger once every 4 hours.* Creating database dumps can be accomplished with the `bluedrop360/postgres-dump-to-s3` repo. From 6308506e2e3fc1800af41c7113ed1446a6a23cb3 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Wed, 17 Jul 2019 00:05:26 -0230 Subject: [PATCH 06/11] fix: removed irrelevant code from the dockerfile --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index e5e022e..61ecd60 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,9 +19,9 @@ RUN apk --no-cache add dumb-init postgresql=${pg_version} python py-pip && \ #-------------------------------------------------------------------------------- # Set script permissions and create required directories #-------------------------------------------------------------------------------- -COPY aws-mfa.sh action.sh / -RUN chmod +x action.sh && chmod +x aws-mfa.sh -RUN mkdir -p /cache && mkdir -p /root/.aws +COPY action.sh / +RUN chmod +x action.sh +RUN mkdir -p /cache #-------------------------------------------------------------------------------- # Use the `dumb-init` init system (PID 1) for process handling From e50689716a0c030e10ef75fdccbc8f0ddd1f9364 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Mon, 19 Aug 2019 14:11:55 -0230 Subject: [PATCH 07/11] fix: use the correct letter case for '$tag' variable --- build_push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_push.sh b/build_push.sh index cbdd0ba..adc4f26 100755 --- a/build_push.sh +++ b/build_push.sh @@ -10,7 +10,7 @@ latest 9.6.10-r0 3.6 # shellcheck disable=SC2039 IFS=$'\n' for build in $builds; do - TAG=$(echo "${build}" | cut -d ' ' -f 1 ) + tag=$(echo "${build}" | cut -d ' ' -f 1 ) pg_version=$(echo "${build}" | cut -d ' ' -f 2) pg_alpine_version=$(echo "${build}" | cut -d ' ' -f 3) From 43dd95dc0df9a06de656bf4681bcd952c1a5273b Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Thu, 12 Sep 2019 13:25:11 -0230 Subject: [PATCH 08/11] system: upgrade postgres --- Dockerfile | 4 ++-- build_push.sh | 24 ++++++++++-------------- docker-compose.yml | 6 +++--- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61ecd60..af5ee1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,11 +8,11 @@ ARG pg_version # Install dependencies #-------------------------------------------------------------------------------- # "postgresql" is required for "pg_restore" -# "python" is required for "aws-cli" +# "py-pip" is required for "aws-cli" #-------------------------------------------------------------------------------- RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${pg_alpine_branch}/main" >> /etc/apk/repositories -RUN apk --no-cache add dumb-init postgresql=${pg_version} python py-pip && \ +RUN apk --no-cache --update add dumb-init postgresql=${pg_version} py-pip && \ pip install awscli && \ apk --purge -v del py-pip diff --git a/build_push.sh b/build_push.sh index adc4f26..1abd758 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,21 +1,17 @@ #!/usr/bin/env sh -builds=$(echo ' -9.6.10 9.6.10-r0 3.6 -9.6 9.6.10-r0 3.6 -9 9.6.10-r0 3.6 -latest 9.6.10-r0 3.6 -' | grep -v '^#' | tr -s ' ') +builds=\ +"10.10_10.10-r0_3.8",\ +"10_10.10-r0_3.8",\ +"latest_10.10-r0_3.8" -# shellcheck disable=SC2039 -IFS=$'\n' -for build in $builds; do - tag=$(echo "${build}" | cut -d ' ' -f 1 ) - pg_version=$(echo "${build}" | cut -d ' ' -f 2) - pg_alpine_version=$(echo "${build}" | cut -d ' ' -f 3) +for build in $(echo $builds | tr ',' '\n'); do + tag=$(echo "${build}" | cut -d '_' -f 1 ) + pg_version=$(echo "${build}" | cut -d '_' -f 2) + pg_alpine_branch=$(echo "${build}" | cut -d '_' -f 3) - echo docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_version}" . - eval docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_version}" . + echo docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_branch}" . + eval docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_branch}" . echo docker push bluedrop360/postgres-restore-from-s3:"${tag}" eval docker push bluedrop360/postgres-restore-from-s3:"${tag}" done diff --git a/docker-compose.yml b/docker-compose.yml index 70c2799..33ab5e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,14 @@ version: '3' services: postgres-restore-from-s3: - image: postgres-restore-from-s3:9.6.10 + image: postgres-restore-from-s3:10.10 network_mode: 'host' build: context: ./ dockerfile: ./Dockerfile args: - pg_alpine_branch: '3.6' - pg_version: '9.6.10-r0' + pg_alpine_branch: '3.8' + pg_version: '10.10-r0' environment: AWS_BUCKET: AWS_REGION: From f85b937682bf7f1405025615bbc5b47345831165 Mon Sep 17 00:00:00 2001 From: "J. Abbott" Date: Mon, 30 Sep 2019 16:08:09 -0230 Subject: [PATCH 09/11] improve the build script --- Dockerfile | 12 ++++++------ build_push.sh | 36 +++++++++++++++++++++++++----------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/Dockerfile b/Dockerfile index af5ee1e..a0da40b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ -ARG pg_alpine_branch -FROM alpine:${pg_alpine_branch} +ARG alpine_version +FROM alpine:${alpine_version} -ARG pg_alpine_branch -ARG pg_version +ARG alpine_version +ARG pg_package_version #-------------------------------------------------------------------------------- # Install dependencies @@ -10,9 +10,9 @@ ARG pg_version # "postgresql" is required for "pg_restore" # "py-pip" is required for "aws-cli" #-------------------------------------------------------------------------------- -RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${pg_alpine_branch}/main" >> /etc/apk/repositories +RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${alpine_version}/main" >> /etc/apk/repositories -RUN apk --no-cache --update add dumb-init postgresql=${pg_version} py-pip && \ +RUN apk --no-cache --update add dumb-init postgresql=${pg_package_version} py-pip && \ pip install awscli && \ apk --purge -v del py-pip diff --git a/build_push.sh b/build_push.sh index 1abd758..f126fa6 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,17 +1,31 @@ #!/usr/bin/env sh +alpine_version='3.8' +legacy_image=false +pg_major_version='10' +pg_package_version='10.10' + builds=\ -"10.10_10.10-r0_3.8",\ -"10_10.10-r0_3.8",\ -"latest_10.10-r0_3.8" +"${pg_package_version}_${pg_package_version}-r0_${alpine_version}",\ +"${pg_major_version}_${pg_package_version}-r0_${alpine_version}" + +if [ "${legacy_image}" = 'false' ]; then + builds="${builds}","latest_${pg_package_version}-r0_${alpine_version}" +fi + +echo $builds | tr ',' '\n' | while read build; do + alpine_version=$(echo "${build}" | cut -d '_' -f 3) + pg_restore_from_s3_tag=$(echo "${build}" | cut -d '_' -f 1 ) + pg_package_version=$(echo "${build}" | cut -d '_' -f 2) -for build in $(echo $builds | tr ',' '\n'); do - tag=$(echo "${build}" | cut -d '_' -f 1 ) - pg_version=$(echo "${build}" | cut -d '_' -f 2) - pg_alpine_branch=$(echo "${build}" | cut -d '_' -f 3) + echo "" + echo "--------------------------------" + echo "POSTGRES-RESTORE-FROM-S3 TAG: ${pg_restore_from_s3_tag}" + echo "POSTGRES PACKAGE VERSION: ${pg_package_version}" + echo "--------------------------------" - echo docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_branch}" . - eval docker build --tag bluedrop360/postgres-restore-from-s3:"${tag}" --build-arg pg_version="${pg_version}" --build-arg pg_alpine_branch="${pg_alpine_branch}" . - echo docker push bluedrop360/postgres-restore-from-s3:"${tag}" - eval docker push bluedrop360/postgres-restore-from-s3:"${tag}" + echo docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_package_version=$pg_package_version --build-arg alpine_version="${alpine_version}" . + eval docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_package_version=$pg_package_version --build-arg alpine_version="${alpine_version}" . || exit 1 + echo docker push bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag + eval docker push bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag || exit 1 done From 7b01d6f62f49ac283933ba899476c9a01f62286e Mon Sep 17 00:00:00 2001 From: Jared Abbott Date: Fri, 15 May 2020 12:13:11 -0230 Subject: [PATCH 10/11] add 11.7 support and improve scripts --- .travis.yml | 2 +- Dockerfile | 4 ++-- README.md | 19 ++++++++++++++++ build_push.sh | 51 +++++++++++++++++++++++++++++++----------- dev/util/shellcheck.sh | 8 +++++++ docker-compose.yml | 6 ++--- lint.sh | 17 -------------- package/10.12-3.8.env | 3 +++ package/11.7-3.9.env | 4 ++++ package/9.6.13-3.6.env | 3 +++ 10 files changed, 81 insertions(+), 36 deletions(-) create mode 100755 dev/util/shellcheck.sh delete mode 100644 lint.sh create mode 100644 package/10.12-3.8.env create mode 100644 package/11.7-3.9.env create mode 100644 package/9.6.13-3.6.env diff --git a/.travis.yml b/.travis.yml index a42996d..8c791e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,4 +29,4 @@ branches: - master script: - - ./lint.sh --sh + - ./dev/util/shellcheck.sh diff --git a/Dockerfile b/Dockerfile index a0da40b..f72cd42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG alpine_version FROM alpine:${alpine_version} ARG alpine_version -ARG pg_package_version +ARG pg_full_version #-------------------------------------------------------------------------------- # Install dependencies @@ -12,7 +12,7 @@ ARG pg_package_version #-------------------------------------------------------------------------------- RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${alpine_version}/main" >> /etc/apk/repositories -RUN apk --no-cache --update add dumb-init postgresql=${pg_package_version} py-pip && \ +RUN apk --no-cache --update add dumb-init postgresql=${pg_full_version} py-pip && \ pip install awscli && \ apk --purge -v del py-pip diff --git a/README.md b/README.md index f1bc80f..78b401d 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,25 @@ Cron based download from s3 and database restore. +## Build + +`./build_push.sh [-p , --package ]` + +`./build_push.sh -p 11.7-3.9` + +### Package files + +Each package file represents a release for a particular `postgres` branch. + +The contents of the latest package file may look like this: + +``` +ALPINE_VERSION='3.9' +PG_BASE_VERSION='11' +PG_FULL_VERSION='11.7' +PG_LATEST=true +``` + ## Usage Typically this image is instantiated as a container among many others and would have the responsibility of getting downloading a dump file from s3 and restoring a database at a particular time of day. diff --git a/build_push.sh b/build_push.sh index f126fa6..b7047f2 100755 --- a/build_push.sh +++ b/build_push.sh @@ -1,31 +1,56 @@ #!/usr/bin/env sh -alpine_version='3.8' -legacy_image=false -pg_major_version='10' -pg_package_version='10.10' +#------------------------------------------------------------------------------------ +# Loop over arguments +#------------------------------------------------------------------------------------ +for arg in "$@"; do + # [ -p | --package ] + if [ -n "${package_flag}" ]; then + package_flag='' + package="${arg}" + fi + + if [ "${arg}" = "-p" ] || [ "${arg}" = "--package" ]; then + package_flag=true + fi +done + +#------------------------------------------------------------------------------------ +# Exit on error +#------------------------------------------------------------------------------------ +if [ -z "${package}" ]; then + echo '> Package file not specified: [-p , --package ]' + exit 127 +fi + +if [ -f "./package/${package}.env" ]; then + . "./package/${package}.env" +else + echo "> Package file not found: './package/${package}.env'" + exit 127 +fi builds=\ -"${pg_package_version}_${pg_package_version}-r0_${alpine_version}",\ -"${pg_major_version}_${pg_package_version}-r0_${alpine_version}" +"${PG_FULL_VERSION}_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}",\ +"${PG_BASE_VERSION}_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}" -if [ "${legacy_image}" = 'false' ]; then - builds="${builds}","latest_${pg_package_version}-r0_${alpine_version}" +if [ "${PG_LATEST:-'false'}" = 'true' ]; then + builds="${builds}","latest_${PG_FULL_VERSION}-r0_${ALPINE_VERSION}" fi echo $builds | tr ',' '\n' | while read build; do - alpine_version=$(echo "${build}" | cut -d '_' -f 3) + ALPINE_VERSION=$(echo "${build}" | cut -d '_' -f 3) pg_restore_from_s3_tag=$(echo "${build}" | cut -d '_' -f 1 ) - pg_package_version=$(echo "${build}" | cut -d '_' -f 2) + PG_FULL_VERSION=$(echo "${build}" | cut -d '_' -f 2) echo "" echo "--------------------------------" echo "POSTGRES-RESTORE-FROM-S3 TAG: ${pg_restore_from_s3_tag}" - echo "POSTGRES PACKAGE VERSION: ${pg_package_version}" + echo "POSTGRES PACKAGE VERSION: ${PG_FULL_VERSION}" echo "--------------------------------" - echo docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_package_version=$pg_package_version --build-arg alpine_version="${alpine_version}" . - eval docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_package_version=$pg_package_version --build-arg alpine_version="${alpine_version}" . || exit 1 + echo docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_full_version=$PG_FULL_VERSION --build-arg alpine_version="${ALPINE_VERSION}" . + eval docker build --tag bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag --build-arg pg_full_version=$PG_FULL_VERSION --build-arg alpine_version="${ALPINE_VERSION}" . || exit 1 echo docker push bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag eval docker push bluedrop360/postgres-restore-from-s3:$pg_restore_from_s3_tag || exit 1 done diff --git a/dev/util/shellcheck.sh b/dev/util/shellcheck.sh new file mode 100755 index 0000000..af7e20d --- /dev/null +++ b/dev/util/shellcheck.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +printf '\n%s\n' "Checking shell scripts..." + +SHELLCHECK_OPTS="" + +RUN_SHELLCHECK="shellcheck ${ALLOW_EXTERNAL_SOURCE:-} ${SHELLCHECK_OPTS} {} +" +eval "find ./*.sh -type f -exec ${RUN_SHELLCHECK}" diff --git a/docker-compose.yml b/docker-compose.yml index 33ab5e3..744e16f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,14 +11,14 @@ version: '3' services: postgres-restore-from-s3: - image: postgres-restore-from-s3:10.10 + image: postgres-restore-from-s3:11.7 network_mode: 'host' build: context: ./ dockerfile: ./Dockerfile args: - pg_alpine_branch: '3.8' - pg_version: '10.10-r0' + alpine_version: '3.9' + pg_full_version: '11.7-r0' environment: AWS_BUCKET: AWS_REGION: diff --git a/lint.sh b/lint.sh deleted file mode 100644 index be6fb52..0000000 --- a/lint.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env sh - -ALLOW_EXTERNAL_SOURCE="${ALLOW_EXTERNAL_SOURCE:-}" - -if [ "${1}" = "--yml" ]; then - printf '\n%s\n' "Checking '.travis.yml'..." - travis lint ./.travis.yml -fi - -if [ "${1}" = "--sh" ]; then - printf '\n%s\n' "Checking shell scripts..." - - SHELLCHECK_OPTS="" - - RUN_SHELLCHECK="shellcheck ${ALLOW_EXTERNAL_SOURCE} ${SHELLCHECK_OPTS} {} +" - eval "find ./*.sh -type f -exec ${RUN_SHELLCHECK}" -fi diff --git a/package/10.12-3.8.env b/package/10.12-3.8.env new file mode 100644 index 0000000..c1f48eb --- /dev/null +++ b/package/10.12-3.8.env @@ -0,0 +1,3 @@ +ALPINE_VERSION='3.8' +PG_BASE_VERSION='10' +PG_FULL_VERSION='10.12' diff --git a/package/11.7-3.9.env b/package/11.7-3.9.env new file mode 100644 index 0000000..970ec5b --- /dev/null +++ b/package/11.7-3.9.env @@ -0,0 +1,4 @@ +ALPINE_VERSION='3.9' +PG_LATEST=false +PG_BASE_VERSION='11' +PG_FULL_VERSION='11.7' diff --git a/package/9.6.13-3.6.env b/package/9.6.13-3.6.env new file mode 100644 index 0000000..9ee2ce9 --- /dev/null +++ b/package/9.6.13-3.6.env @@ -0,0 +1,3 @@ +ALPINE_VERSION='3.6' +PG_BASE_VERSION='9' +PG_FULL_VERSION='9.6.13' From 453cac06c7e44b57cfc529ce21add1a136b22b58 Mon Sep 17 00:00:00 2001 From: Jared Abbott Date: Tue, 15 Sep 2020 12:25:24 -0230 Subject: [PATCH 11/11] Use Postgres 12.4 --- Dockerfile | 6 +----- README.md | 8 ++++---- action.sh | 4 ++-- docker-compose.yml | 4 ++-- package/12.4-3.12.env | 4 ++++ 5 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 package/12.4-3.12.env diff --git a/Dockerfile b/Dockerfile index f72cd42..3b1a02a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,13 +8,9 @@ ARG pg_full_version # Install dependencies #-------------------------------------------------------------------------------- # "postgresql" is required for "pg_restore" -# "py-pip" is required for "aws-cli" #-------------------------------------------------------------------------------- RUN echo "http://dl-cdn.alpinelinux.org/alpine/v${alpine_version}/main" >> /etc/apk/repositories - -RUN apk --no-cache --update add dumb-init postgresql=${pg_full_version} py-pip && \ - pip install awscli && \ - apk --purge -v del py-pip +RUN apk --no-cache --update add dumb-init postgresql=${pg_full_version} aws-cli #-------------------------------------------------------------------------------- # Set script permissions and create required directories diff --git a/README.md b/README.md index 78b401d..44f871f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Cron based download from s3 and database restore. `./build_push.sh [-p , --package ]` -`./build_push.sh -p 11.7-3.9` +`./build_push.sh -p 12.4-3.12` ### Package files @@ -15,9 +15,9 @@ Each package file represents a release for a particular `postgres` branch. The contents of the latest package file may look like this: ``` -ALPINE_VERSION='3.9' -PG_BASE_VERSION='11' -PG_FULL_VERSION='11.7' +ALPINE_VERSION='3.12' +PG_BASE_VERSION='12' +PG_FULL_VERSION='12.4' PG_LATEST=true ``` diff --git a/action.sh b/action.sh index b6bfbfe..53bcf7c 100755 --- a/action.sh +++ b/action.sh @@ -100,10 +100,10 @@ if [ -n "$PRE_RESTORE_PSQL" ]; then fi if [ -n "$SCHEMA" ]; then - printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d /cache/${dump_file}" + printf '\n%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --schema $SCHEMA --no-owner -d /cache/${dump_file}" pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --schema "$SCHEMA" --no-owner -d "${DATABASE_URL}" "/cache/${dump_file}" else - printf '%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d /cache/${dump_file}" + printf '\n%s' " pg_restore --jobs $(grep -c ^processor /proc/cpuinfo) --no-owner -d /cache/${dump_file}" pg_restore --jobs "$(grep -c ^processor /proc/cpuinfo)" --no-owner -d "${DATABASE_URL}" "/cache/${dump_file}" fi diff --git a/docker-compose.yml b/docker-compose.yml index 744e16f..8d2bc99 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,8 @@ services: context: ./ dockerfile: ./Dockerfile args: - alpine_version: '3.9' - pg_full_version: '11.7-r0' + alpine_version: '3.12' + pg_full_version: '12.4-r0' environment: AWS_BUCKET: AWS_REGION: diff --git a/package/12.4-3.12.env b/package/12.4-3.12.env new file mode 100644 index 0000000..806b28e --- /dev/null +++ b/package/12.4-3.12.env @@ -0,0 +1,4 @@ +ALPINE_VERSION='3.12' +PG_LATEST=false +PG_BASE_VERSION='12' +PG_FULL_VERSION='12.4'