Merge pull request #3 from zalando/s3_backups

S3 backups

postgres-appliance/Dockerfile

@@ -58,6 +58,10 @@ ENV ETCD_DISCOVERY_URL postgres.acid.example.com
 ENV SCOPE test
 ENV WAL_S3_BUCKET spilo-example-com
 ENV DEBUG 0
+# Shoud be 3am by GMT+2 aka Berlin Summer Time
+ENV BACKUP_HOUR 1
+ENV WALE_BACKUP_THRESHOLD_MEGABYTES 1024
+ENV WALE_BACKUP_THRESHOLD_PERCENTAGE 30
 ENV LC_ALL en_US.utf-8
 # run subsequent commands as user postgres
 USER postgres

postgres-appliance/postgres_ha.sh

@@ -5,6 +5,7 @@ WALE_ENV_DIR=/home/postgres/etc/wal-e.d/env
 
 SSL_CERTIFICATE="/home/postgres/dummy.crt"
 SSL_PRIVATE_KEY="/home/postgres/dummy.key"
+BACKUP_INTERVAL=3600
 
 function write_postgres_yaml
 {
@@ -29,6 +30,10 @@ postgresql:
   admin:
     username: admin
     password: admin
+  wal_e:
+    env_dir: $WALE_ENV_DIR
+    threshold_megabytes: ${WALE_BACKUP_THRESHOLD_MEGABYTES}
+    threshold_backup_size_percentage: ${WALE_BACKUP_THRESHOLD_PERCENTAGE}
   parameters:
     archive_mode: "on"
     wal_level: hot_standby
@@ -63,7 +68,74 @@ write_archive_command_environment
 # for the -proxy on TDB the url of the etcd cluster
 [ "$DEBUG" -eq 1 ] && exec /bin/bash
 
-etcd -name "proxy-$SCOPE" -proxy on --data-dir=etcd -discovery-srv $ETCD_DISCOVERY_URL &
+# resurrect etcd if it's gone
+(
+  while true
+  do
+    etcd -name "proxy-$SCOPE" -proxy on  --data-dir=etcd -discovery-srv $ETCD_DISCOVERY_URL
+  done
+) &
+
+# run wal-e s3 backup periodically
+# XXX: for debugging purposes, it's running every 5 minutes
+(
+  INITIAL=1
+  RETRY=0
+  LAST_BACKUP_TS=0
+  while true
+  do
+    sleep 5
+
+    CURRENT_TS=$(date +%s)
+    CURRENT_HOUR=$(date +%H)
+    pg_isready >/dev/null 2>&2 || continue
+    IN_RECOVERY=$(psql -tqAc "select pg_is_in_recovery()")
+
+    [[ $IN_RECOVERY != "f" ]] && echo "still in recovery" && continue
+    # during initial run, count the number of backup lines. If there are
+    # no backup (only line with backup-list header is returned), or there
+    # is an error, try to produce a backup. Otherwise, stick to the regular
+    # schedule, since we might run the backups on a freshly promoted replica.
+    if [[ $INITIAL = 1 ]]
+    then
+      BACKUPS_LINES=$(envdir ${WALE_ENV_DIR} wal-e --aws-instance-profile  backup-list 2>/dev/null|wc -l)
+      [[ $PIPESTATUS[0] = 0 ]] && [[ $BACKUPS_LINES -ge 2 ]] && INITIAL=0
+    fi
+    # produce backup only at a given hour, unless it's set to *, which means
+    # that only backup_interval is taken into account. We also skip all checks
+    # when the backup is forced because of previous attempt's failure or because
+    # it's going to be a very first backup, in which case we create it unconditionally.
+    if [[ $RETRY = 0 ]] && [[ $INITIAL = 0 ]]
+    then
+      # check that enough time has passed since the previous backup
+      [[ $BACKUP_HOUR != '*' ]] && [[ $CURRENT_HOUR != $BACKUP_HOUR ]] && continue
+      # get the time since the last backup. Do it only one when the hour
+      # matches the backup hour.
+      [[ $LAST_BACKUP_TS = 0 ]] && LAST_BACKUP_TS=$(envdir ${WALE_ENV_DIR} wal-e --aws-instance-profile  backup-list LATEST 2>/dev/null | tail -n1 | awk '{print $2}' | xargs date +%s --date)
+      # LAST_BACKUP_TS will be empty on error.
+      if [[ -z $LAST_BACKUP_TS ]]
+      then
+        LAST_BACKUP_TS=0
+        echo "could not obtain latest backup timestamp"
+      fi
+
+      ELAPSED_TIME=$((CURRENT_TS-LAST_BACKUP_TS))
+      [[ $ELAPSED_TIME -lt $BACKUP_INTERVAL ]] && continue
+    fi
+    # leave only 2 base backups before creating a new one
+    envdir ${WALE_ENV_DIR} wal-e --aws-instance-profile delete --confirm retain 2
+    # push a new base backup
+    echo "producing a new backup at $(date)"
+    envdir ${WALE_ENV_DIR} wal-e --aws-instance-profile backup-push ${PGDATA}
+    RETRY=$?
+    # re-examine last backup timestamp if a new backup has been created
+    if [[ $RETRY = 0 ]]
+    then
+      INITIAL=0
+      LAST_BACKUP_TS=0
+    fi
+  done
+) &
 
 exec governor/governor.py "/home/postgres/postgres.yml"