Skip to content
Permalink
Browse files

feat(postgres): adding upgrade scripts to the postgres

  • Loading branch information...
duanhongyi
duanhongyi committed Dec 1, 2018
1 parent d3a7f62 commit ae89bf46d77fb3fd798b1361f4709793e5d029e3
@@ -1,9 +1,10 @@
FROM postgres:9.4-alpine
FROM postgres:11-alpine

ENV WALE_LOG_DESTINATION stderr
ENV WALE_ENVDIR /etc/wal-e.d/env
ENV PGDATA_RECOVERY /var/lib/postgresql/recovery

RUN mkdir -p $WALE_ENVDIR \
RUN mkdir -p $WALE_ENVDIR $PGDATA_RECOVERY \
&& echo 'http://dl-cdn.alpinelinux.org/alpine/v3.5/main' >> /etc/apk/repositories \
&& apk add --update --virtual .build-deps \
git \
@@ -31,6 +32,8 @@ RUN mkdir -p $WALE_ENVDIR \

COPY rootfs /

VOLUME /var/lib/postgresql/recovery

ARG PATCH_CMD="python3 /patcher-script.py"
RUN $PATCH_CMD file /bin/create_bucket /patcher-script.d/patch_boto_s3.py
RUN $PATCH_CMD module wal_e.cmd /patcher-script.d/patch_boto_s3.py
@@ -8,7 +8,7 @@ IMAGE_PREFIX ?= deis

include versioning.mk

SHELL_SCRIPTS = $(wildcard _scripts/*.sh contrib/ci/*.sh rootfs/bin/*backup) rootfs/bin/is_running
SHELL_SCRIPTS = $(wildcard _scripts/*.sh contrib/ci/*.sh rootfs/bin/*backup rootfs/bin/do_upgrade) rootfs/bin/is_running

# The following variables describe the containerized development environment
# and other build options
@@ -30,7 +30,10 @@ test: test-style test-functional
test-style:
${DEV_ENV_CMD} shellcheck $(SHELL_SCRIPTS)

test-functional: test-functional-swift test-functional-minio
test-functional: test-functional-upgrade test-functional-swift test-functional-minio

test-functional-upgrade:
contrib/ci/test-upgrade.sh ${IMAGE}

test-functional-minio:
contrib/ci/test-minio.sh ${IMAGE}
@@ -0,0 +1,129 @@
#!/usr/bin/env bash

#set -eof pipefail

cleanup() {
kill-containers "${SWIFT_DATA}" "${SWIFT_JOB}" "${PG_JOB}"
}
trap cleanup EXIT

waitting-install-apk() {
sleep 30s
apk_running="apk"
puts-step "Installing apk, coffee"
while true
do
apk_running=$(docker exec "${PG_JOB}" "ps -ef|grep apk|grep -v grep")
if [ -z "$apk_running" ] ; then
break
fi
puts-step "..."
sleep 10s
done
echo "Install apk process is complete"
}

TEST_ROOT=$(dirname "${BASH_SOURCE[0]}")/
# shellcheck source=/dev/null
source "${TEST_ROOT}/test.sh"

# make sure we are in this dir
CURRENT_DIR=$(cd "$(dirname "$0")"|| exit; pwd)

create-postgres-creds

puts-step "fetching openstack credentials"

# turn creds into something that we can use.
mkdir -p "${CURRENT_DIR}"/tmp/swift

# guess which value to use for tenant:
TENANT=""

echo "test:tester" > "${CURRENT_DIR}"/tmp/swift/username
echo "testing" > "${CURRENT_DIR}"/tmp/swift/password
echo "${TENANT}" > "${CURRENT_DIR}"/tmp/swift/tenant
echo "http://swift:8080/auth/v1.0" > "${CURRENT_DIR}"/tmp/swift/authurl
echo "1" > "${CURRENT_DIR}"/tmp/swift/authversion
echo "deis-swift-test" > "${CURRENT_DIR}"/tmp/swift/database-container

# boot swift
SWIFT_DATA=$(docker run -d -v /srv --name SWIFT_DATA busybox)

SWIFT_JOB=$(docker run -d --name onlyone --hostname onlyone --volumes-from SWIFT_DATA -t deis/swift-onlyone:git-8516d23)

test-upgrade-from(){
PGDATA_DIR=${CURRENT_DIR}/tmp/postgres/tmp_$(date +%s)
mkdir -p "$PGDATA_DIR"

docker run --rm \
-v "${PGDATA_DIR}:/var/lib/postgres/pg_data" \
-e PGDATA=/var/lib/postgres/pg_data \
"$1" \
bash -c "chown -R postgres /var/lib/postgres/pg_data && su-exec postgres initdb"

PG_CMD="docker run -d --link ${SWIFT_JOB}:swift -e BACKUP_FREQUENCY=3s \
-e DATABASE_STORAGE=swift \
-e PGCTLTIMEOUT=1200 \
-e PGDATA=/var/lib/postgres/pg_data \
-v ${PGDATA_DIR}:/var/lib/postgres/pg_data \
-v ${CURRENT_DIR}/tmp/creds:/var/run/secrets/deis/database/creds \
-v ${CURRENT_DIR}/tmp/swift:/var/run/secrets/deis/objectstore/creds \
$IMAGE"
PG_JOB=$($PG_CMD)
sleep 90s
puts-step "sleeping for 90s while postgres is restore..."

check-postgres "${PG_JOB}"
puts-step "postgres upgrade from $1"
}

test-upgrade-from-wal() {
PG_CMD="docker run -d --link ${SWIFT_JOB}:swift -e BACKUP_FREQUENCY=3s \
-e DATABASE_STORAGE=swift \
-e PGCTLTIMEOUT=1200 \
-v ${CURRENT_DIR}/tmp/creds:/var/run/secrets/deis/database/creds \
-v ${CURRENT_DIR}/tmp/swift:/var/run/secrets/deis/objectstore/creds \
$1"

start-postgres "$PG_CMD"
# display logs for debugging purposes
puts-step "displaying swift logs"
docker logs "${SWIFT_JOB}"
check-postgres "${PG_JOB}"
puts-step "shutting off postgres, then rebooting to test data recovery"
kill-containers "${PG_JOB}"

PG_CMD="docker run -d --link ${SWIFT_JOB}:swift -e BACKUP_FREQUENCY=3s \
-e DATABASE_STORAGE=swift \
-e PGCTLTIMEOUT=1200 \
-v ${CURRENT_DIR}/tmp/creds:/var/run/secrets/deis/database/creds \
-v ${CURRENT_DIR}/tmp/swift:/var/run/secrets/deis/objectstore/creds \
$IMAGE"

start-postgres "${PG_CMD}"

check-postgres "${PG_JOB}"
}

IMAGE="$1"

test-upgrade-from postgres:9.4-alpine
kill-containers "${PG_JOB}"

test-upgrade-from postgres:9.5-alpine
kill-containers "${PG_JOB}"

test-upgrade-from postgres:9.6-alpine
kill-containers "${PG_JOB}"

test-upgrade-from postgres:10-alpine
kill-containers "${PG_JOB}"

test-upgrade-from postgres:11-alpine
kill-containers "${PG_JOB}"

test-upgrade-from-wal hephy/postgres:v2.7.1

puts-step "tests PASSED!"
exit 0
@@ -0,0 +1,57 @@
#!/usr/bin/env bash

POSTGRES_OLD_VERSION=$(cat "$1"/PG_VERSION)

install-old-postgres() {
if [ "$POSTGRES_OLD_VERSION" == "9.4" ];then
echo 'http://dl-cdn.alpinelinux.org/alpine/v3.3/main' >> /etc/apk/repositories
apk add --update --virtual .upgrade-deps postgresql=9.4.15-r0
elif [ "$POSTGRES_OLD_VERSION" == "9.5" ];then
echo 'http://dl-cdn.alpinelinux.org/alpine/v3.4/main' >> /etc/apk/repositories
apk add --update --virtual .upgrade-deps postgresql=9.5.13-r0
elif [ "$POSTGRES_OLD_VERSION" == "9.6" ];then
echo 'http://dl-cdn.alpinelinux.org/alpine/v3.6/main' >> /etc/apk/repositories
apk add --update --virtual .upgrade-deps postgresql=9.6.10-r0
elif [ "$POSTGRES_OLD_VERSION" == "10" ];then
echo 'http://dl-cdn.alpinelinux.org/alpine/v3.8/main' >> /etc/apk/repositories
apk add --update --virtual .upgrade-deps postgresql=10.5-r0
else
echo "Upgrading from ${POSTGRES_OLD_VERSION} is not supported"
exit 1
fi
}

remove-old-postgres() {
apk del .upgrade-deps
}

if [ "$POSTGRES_OLD_VERSION" != "$PG_MAJOR" ] ; then
echo
echo 'PostgreSQL executing the pg_upgrade script.'
echo

cd /tmp||exit
install-old-postgres

su-exec postgres /usr/bin/pg_ctl \
-D "$1" \
-o "-c listen_addresses=''" \
-w start

su-exec postgres /usr/bin/pg_ctl \
-D "$1" \
-w stop

su-exec postgres pg_upgrade \
--old-datadir="$1" \
--new-datadir="$2" \
--old-bindir=/usr/bin \
--new-bindir=/usr/local/bin

remove-old-postgres
rm -rf /var/cache/apk/*

echo
echo 'PostgreSQL upgrade process complete; ready for start up.'
echo
fi
@@ -20,9 +20,8 @@ su-exec postgres pg_ctl -D "$PGDATA" -w restart
if [[ $(envdir "$WALE_ENVDIR" wal-e --terse backup-list | wc -l) -gt "1" ]]; then
echo "Found backups. Restoring from backup..."
su-exec postgres pg_ctl -D "$PGDATA" -w stop
rm -rf "$PGDATA/*"
envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA" LATEST
cat << EOF > "$PGDATA/postgresql.conf"
envdir "$WALE_ENVDIR" wal-e backup-fetch "$PGDATA_RECOVERY" LATEST
cat << EOF > "$PGDATA_RECOVERY/postgresql.conf"
# These settings are initialized by initdb, but they can be changed.
log_timezone = 'UTC'
lc_messages = 'C' # locale for system error message
@@ -35,8 +34,9 @@ archive_mode = on
archive_command = 'envdir "${WALE_ENVDIR}" wal-e wal-push %p'
archive_timeout = 60
listen_addresses = '*'
unix_socket_directories = '/var/run/postgresql'
EOF
cat << EOF > "$PGDATA/pg_hba.conf"
cat << EOF > "$PGDATA_RECOVERY/pg_hba.conf"
# "local" is for Unix domain socket connections only
local all all trust
# IPv4 local connections:
@@ -46,13 +46,22 @@ host all all ::1/128 trust
# IPv4 global connections
host all all 0.0.0.0/0 md5
EOF
touch "$PGDATA/pg_ident.conf"
echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA/recovery.conf"
chown -R postgres:postgres "$PGDATA"
chmod 0700 "$PGDATA"
su-exec postgres pg_ctl -D "$PGDATA" \
-o "-c listen_addresses=''" \
-w start
touch "$PGDATA_RECOVERY/pg_ident.conf"
echo "restore_command = 'envdir /etc/wal-e.d/env wal-e wal-fetch \"%f\" \"%p\"'" >> "$PGDATA_RECOVERY/recovery.conf"
chown -R postgres:postgres "$PGDATA_RECOVERY"
chmod 0700 "$PGDATA_RECOVERY"

POSTGRES_OLD_VERSION=$(cat "$PGDATA_RECOVERY"/PG_VERSION)
if [ "$POSTGRES_OLD_VERSION" = "$PG_MAJOR" ] ; then
su-exec postgres rm -rf "$PGDATA"/*
su-exec postgres cp -rf "$PGDATA_RECOVERY"/* "$PGDATA"
else
do_upgrade "$PGDATA_RECOVERY" "$PGDATA"
fi
su-exec postgres pg_ctl \
-D "$PGDATA" \
-o "-c listen_addresses=''" \
-w start
fi

# ensure $PGDATA has the right permissions
@@ -14,9 +14,9 @@ POSTGRES_USER="$(cat /var/run/secrets/deis/database/creds/user)"
POSTGRES_PASSWORD="$(cat /var/run/secrets/deis/database/creds/password)"

if [ "$1" = 'postgres' ]; then
mkdir -p "$PGDATA"
chmod 700 "$PGDATA"
chown -R postgres "$PGDATA"
mkdir -p "$PGDATA" "$PGDATA_RECOVERY"
chmod 700 "$PGDATA" "$PGDATA_RECOVERY"
chown -R postgres "$PGDATA" "$PGDATA_RECOVERY"

chmod g+s /run/postgresql
chown -R postgres /run/postgresql
@@ -57,7 +57,23 @@ if [ "$1" = 'postgres' ]; then
su-exec postgres pg_ctl -D "$PGDATA" \
-o "-c listen_addresses=''" \
-w start
echo

echo
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql)
echo "$0: running $f";
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < "$f"
echo
;;
*) echo "$0: ignoring $f" ;;
esac
echo
done

echo
: ${POSTGRES_USER:=postgres}
: ${POSTGRES_DB:=$POSTGRES_USER}
export POSTGRES_USER POSTGRES_DB
@@ -80,26 +96,20 @@ if [ "$1" = 'postgres' ]; then
EOSQL
echo

echo
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql)
echo "$0: running $f";
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" < "$f"
echo
;;
*) echo "$0: ignoring $f" ;;
esac
echo
done

su-exec postgres pg_ctl -D "$PGDATA" -m fast -w stop
set_listen_addresses '*'

echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
else
POSTGRES_OLD_VERSION=$(cat $PGDATA/PG_VERSION)
if [ "$POSTGRES_OLD_VERSION" != "$PG_MAJOR" ] ; then
su-exec postgres cp -rf "$PGDATA"/* "$PGDATA_RECOVERY"
su-exec postgres rm -rf "$PGDATA"/*
su-exec postgres initdb
do_upgrade "$PGDATA_RECOVERY" "$PGDATA"
fi
fi

exec su-exec postgres "$@"

0 comments on commit ae89bf4

Please sign in to comment.
You can’t perform that action at this time.