ghe-backup

#!/usr/bin/env bash
#/ Usage: ghe-backup [-hv] [--version]
#/
#/ Take snapshots of all GitHub Enterprise data, including Git repository data,
#/ the MySQL database, instance settings, GitHub Pages data, etc.
#/
#/ OPTIONS:
#/   -v | --verbose    Enable verbose output.
#/   -h | --help       Show this message.
#/        --version    Display version information.
#/

set -e

# Parse arguments
while true; do
  case "$1" in
    -h|--help)
      export GHE_SHOW_HELP=true
      shift
      ;;
    --version)
      export GHE_SHOW_VERSION=true
      shift
      ;;
    -v|--verbose)
      export GHE_VERBOSE=true
      shift
      ;;
    -*)
      echo "Error: invalid argument: '$1'" 1>&2
      exit 1
      ;;
    *)
      break
      ;;
  esac
done

# Bring in the backup configuration
# shellcheck source=share/github-backup-utils/ghe-backup-config
. "$( dirname "${BASH_SOURCE[0]}" )/../share/github-backup-utils/ghe-backup-config"

# Used to record failed backup steps
failures=

# CPU and IO throttling to keep backups from thrashing around.
export GHE_NICE=${GHE_NICE:-"nice -n 19"}
export GHE_IONICE=${GHE_IONICE:-"ionice -c 3"}

# Create the timestamped snapshot directory where files for this run will live,
# change into it, and mark the snapshot as incomplete by touching the
# 'incomplete' file. If the backup succeeds, this file will be removed
# signifying that the snapshot is complete.
mkdir -p "$GHE_SNAPSHOT_DIR"
cd "$GHE_SNAPSHOT_DIR"
touch "incomplete"

# Exit early if the snapshot filesystem doesn't support hard links, symlinks and
# if rsync doesn't support hardlinking of dangling symlinks
trap 'rm -rf src dest1 dest2' EXIT
mkdir src
touch src/testfile
if ! ln -s /data/does/not/exist/hooks/ src/ >/dev/null 2>&1; then
  echo "Error: the filesystem containing $GHE_DATA_DIR does not support symbolic links." 1>&2
  echo "Git repositories contain symbolic links that need to be preserved during a backup." 1>&2
  exit 1
fi

if ! output=$(rsync -a src/ dest1 2>&1 && rsync -av src/ --link-dest=../dest1 dest2 2>&1); then
  echo "Error: rsync encountered an error that could indicate a problem with permissions," 1>&2
  echo "hard links, symbolic links, or another issue that may affect backups." 1>&2
  echo "$output"
  exit 1
fi

if [ "$(ls -il dest1/testfile | awk '{ print $1 }')" != "$(ls -il dest2/testfile | awk '{ print $1 }')" ]; then
  echo "Error: the filesystem containing $GHE_DATA_DIR does not support hard links." 1>&2
  echo "Backup Utilities use hard links to store backup data efficiently." 1>&2
  exit 1
fi
rm -rf src dest1 dest2

# To prevent multiple backup runs happening at the same time, we create a
# in-progress file with the timestamp and pid of the backup process,
# giving us a form of locking.
#
# Set up a trap to remove the in-progress file if we exit for any reason but
# verify that we are the same process before doing so.
#
# The cleanup trap also handles disabling maintenance mode on the appliance if
# it was automatically enabled.
cleanup () {
  if [ -f ../in-progress ]; then
    progress=$(cat ../in-progress)
    snapshot=$(echo "$progress" | cut -d ' ' -f 1)
    pid=$(echo "$progress" | cut -d ' ' -f 2)
    if [ "$snapshot" = "$GHE_SNAPSHOT_TIMESTAMP" ] && [ "$$" = $pid ]; then
      unlink ../in-progress
    fi
  fi

  # Cleanup SSH multiplexing
  ghe-ssh --clean "$GHE_HOSTNAME"
}

# Setup exit traps
trap 'cleanup' EXIT
trap 'exit $?' INT # ^C always terminate

if [ -h ../in-progress ]; then
  echo "Error: detected a backup already in progress from a previous version of ghe-backup." 1>&2
  echo "If there is no backup in progress anymore, please remove" 1>&2
  echo "the $GHE_DATA_DIR/in-progress file." 1>&2
  exit 1
fi

if [ -f ../in-progress ]; then
  progress=$(cat ../in-progress)
  snapshot=$(echo "$progress" | cut -d ' ' -f 1)
  pid=$(echo "$progress" | cut -d ' ' -f 2)
  if ! ps -p $pid | grep ghe-backup; then
    # We can safely remove in-progress, ghe-prune-snapshots
    # will clean up the failed backup.
    unlink ../in-progress
  else
    echo "Error: backup process $pid of $GHE_HOSTNAME already in progress in snapshot $snapshot. Aborting." 1>&2
    exit 1
  fi
fi

echo "$GHE_SNAPSHOT_TIMESTAMP $$" > ../in-progress

echo "Starting backup of $GHE_HOSTNAME with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP"

# Perform a host connection check and establish the remote appliance version.
# The version is available in the GHE_REMOTE_VERSION variable and also written
# to a version file in the snapshot directory itself.
ghe_remote_version_required
echo "$GHE_REMOTE_VERSION" > version

if [ -n "$GHE_ALLOW_REPLICA_BACKUP" ]; then
  echo "Warning: backing up a high availability replica may result in inconsistent or unreliable backups."
fi

# Log backup start message in /var/log/syslog on remote instance
ghe_remote_logger "Starting backup from $(hostname) with backup-utils v$BACKUP_UTILS_VERSION in snapshot $GHE_SNAPSHOT_TIMESTAMP ..."

export GHE_BACKUP_STRATEGY=${GHE_BACKUP_STRATEGY:-$(ghe-backup-strategy)}

# Record the strategy with the snapshot so we will know how to restore.
echo "$GHE_BACKUP_STRATEGY" > strategy

# Create benchmark file
bm_init > /dev/null

ghe-backup-store-version  ||
echo "Warning: storing backup-utils version remotely failed."

echo "Backing up GitHub settings ..."
ghe-backup-settings || failures="$failures settings"

echo "Backing up SSH authorized keys ..."
bm_start "ghe-export-authorized-keys"
ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-authorized-keys' > authorized-keys.json ||
failures="$failures authorized-keys"
bm_end "ghe-export-authorized-keys"

echo "Backing up SSH host keys ..."
bm_start "ghe-export-ssh-host-keys"
ghe-ssh "$GHE_HOSTNAME" -- 'ghe-export-ssh-host-keys' > ssh-host-keys.tar ||
failures="$failures ssh-host-keys"
bm_end "ghe-export-ssh-host-keys"

echo "Backing up MySQL database ..."
bm_start "ghe-export-mysql"
echo 'set -o pipefail; ghe-export-mysql | gzip' |
ghe-ssh "$GHE_HOSTNAME" -- /bin/bash > mysql.sql.gz || failures="$failures mysql"
bm_end "ghe-export-mysql"

echo "Backing up Redis database ..."
ghe-backup-redis > redis.rdb || failures="$failures redis"

echo "Backing up audit log ..."
ghe-backup-es-audit-log || failures="$failures audit-log"

echo "Backing up hookshot logs ..."
ghe-backup-es-hookshot || failures="$failures hookshot"

echo "Backing up Git repositories ..."
ghe-backup-repositories || failures="$failures repositories"

echo "Backing up GitHub Pages ..."
ghe-backup-pages || failures="$failures pages"

echo "Backing up storage data ..."
ghe-backup-storage || failures="$failures storage"

echo "Backing up custom Git hooks ..."
ghe-backup-git-hooks || failures="$failures git-hooks"

if [ "$GHE_BACKUP_STRATEGY" = "rsync" ]; then
  echo "Backing up Elasticsearch indices ..."
  ghe-backup-es-rsync || failures="$failures elasticsearch"
fi

# git fsck repositories after the backup
if [ "$GHE_BACKUP_FSCK" = "yes" ]; then
  ghe-backup-fsck $GHE_SNAPSHOT_DIR || failures="$failures fsck"
fi

# If everything was successful, mark the snapshot as complete, update the
# current symlink to point to the new snapshot and prune expired and failed
# snapshots.
if [ -z "$failures" ]; then
  rm "incomplete"

  rm -f "../current"
  ln -s "$GHE_SNAPSHOT_TIMESTAMP" "../current"

  ghe-prune-snapshots
fi

echo "Completed backup of $GHE_HOSTNAME in snapshot $GHE_SNAPSHOT_TIMESTAMP at $(date +"%H:%M:%S")"

# Exit non-zero and list the steps that failed.
if [ -z "$failures" ]; then
  ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP successfully."
else
  steps="$(echo $failures | sed 's/ /, /g')"
  ghe_remote_logger "Completed backup from $(hostname) / snapshot $GHE_SNAPSHOT_TIMESTAMP with failures: ${steps}."
  echo "Error: Snapshot incomplete. Some steps failed: ${steps}. "
  exit 1
fi

# Detect if the created backup contains any leaked ssh keys
echo "Checking for leaked ssh keys ..."
ghe-detect-leaked-ssh-keys -s "$GHE_SNAPSHOT_DIR" || true

# Make sure we exit zero after the conditional
true