Merge branch 'master' into lildude/clean-old-nodes-on-restore

share/github-backup-utils/ghe-backup-repositories

@@ -164,6 +164,7 @@ rsync_repository_data () {
       --rsync-path='sudo -u git rsync' \
       --include-from=- --exclude=\* \
       --files-from="$files_list" \
+      --ignore-missing-args \
       "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
       "$backup_dir" 1>&3
   else
@@ -173,6 +174,7 @@ rsync_repository_data () {
       $link_dest "$@" \
       --rsync-path='sudo -u git rsync' \
       --include-from=- --exclude=\* \
+      --ignore-missing-args \
       "$host:$GHE_REMOTE_DATA_USER_DIR/repositories/" \
       "$backup_dir" 1>&3
   fi
@@ -362,4 +364,15 @@ RULES
 done
 bm_end "$(basename $0) - Special Data Directories Sync"
 
+if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
+  bm_start "$(basename $0) - Verifying Routes"
+
+  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 5 -maxdepth 6 -type d -name \*.git -exec dirname {} \; | sort | uniq) > $tempdir/destination_routes
+
+  git diff -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more repository networks and/or gists were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
+
+  bm_end "$(basename $0) - Verifying Routes"
+fi
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-backup-storage

@@ -126,6 +126,7 @@ for file_list in $tempdir/*.rsync; do
   $link_dest "$@" \
   --rsync-path='sudo -u git rsync' \
   --files-from="$file_list" \
+  --ignore-missing-args \
   --size-only \
   "$hostname:$GHE_REMOTE_DATA_USER_DIR/storage/" \
   "$backup_dir" 1>&3 &
@@ -136,4 +137,15 @@ for pid in $(jobs -p); do
 done
 bm_end "$(basename $0) - Storage object sync"
 
+if [ -z "$GHE_SKIP_ROUTE_VERIFICATION" ]; then
+  bm_start "$(basename $0) - Verifying Routes"
+
+  cat $tempdir/*.rsync | sort | uniq > $tempdir/source_routes
+  (cd $backup_dir/ && find * -mindepth 3 -maxdepth 3 -type f -print | sort | uniq) > $tempdir/destination_routes
+
+  git diff -- $tempdir/source_routes $tempdir/destination_routes || echo "Warning: One or more storage objects were not found on the source appliance. Please contact GitHub Enterprise Support for assistance."
+
+  bm_end "$(basename $0) - Verifying Routes"
+fi
+
 bm_end "$(basename $0)"

share/github-backup-utils/ghe-rsync

@@ -14,15 +14,30 @@ set -o pipefail
 # Filter vanished file warnings from both stdout (rsync versions < 3.x) and
 # stderr (rsync versions >= 3.x). The complex redirections are necessary to
 # filter stderr while also keeping stdout and stderr separated.
-IGNOREOUT='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
-(rsync "${@}" $GHE_EXTRA_RSYNC_OPTS 3>&1 1>&2 2>&3 3>&- |
-  (egrep -v "$IGNOREOUT" || true)) 3>&1 1>&2 2>&3 3>&- |
-  (egrep -v "$IGNOREOUT" || true)
+ignoreout='^(file has vanished: |rsync warning: some files vanished before they could be transferred)'
+
+# Check for --ignore-missing-args parameter support and remove if unavailable.
+if rsync -h | grep '\-\-ignore-missing-args' >/dev/null 2>&1; then
+  parameters=("$@")
+else
+  for parameter in "$@"; do
+    [[ ! $parameter == "--ignore-missing-args" ]] && parameters+=("$parameter") || ignore23=1
+  done
+fi
+
+(rsync "${parameters[@]}" $GHE_EXTRA_RSYNC_OPTS 3>&1 1>&2 2>&3 3>&- |
+  (egrep -v "$ignoreout" || true)) 3>&1 1>&2 2>&3 3>&- |
+  (egrep -v "$ignoreout" || true)
 res=$?
 
-# rsync exits with 24 when vanished files are detected.
+# Suppress exits with 24.
 if [ $res = 24 ]; then
   res=0
 fi
 
+# Suppress exits with 23 if --ignore-missing-args was unavailable.
+if [ $res = 23 ] && [ -n "$ignore23" ]; then
+  res=0
+fi
+
 exit $res

test/test-ghe-backup.sh

@@ -300,3 +300,47 @@ begin_test "ghe-backup cluster"
   verify_all_backedup_data
 )
 end_test
+
+begin_test "ghe-backup not missing directories or files on source appliance"
+(
+    # Tests the scenario where the database and on disk state are consistent.
+    set -e
+
+    if ! ghe-backup -v > "$TRASHDIR/backup-out" 2>&1; then
+      cat "$TRASHDIR/backup-out"
+      : ghe-backup should have completed successfully
+      false
+    fi
+
+    # Ensure the output doesn't contain the warnings
+    grep -q "Warning: One or more repository networks and/or gists were not found on the source appliance." "$TRASHDIR/backup-out" && exit 1
+    grep -q "Warning: One or more storage objects were not found on the source appliance." "$TRASHDIR/backup-out" && exit 1
+
+    verify_all_backedup_data
+)
+end_test
+
+begin_test "ghe-backup missing directories or files on source appliance"
+(
+    # Tests the scenario where something exists in the database, but not on disk.
+    set -e
+
+    rm -rf $GHE_REMOTE_DATA_USER_DIR/repositories/1
+    rm -rf $GHE_REMOTE_DATA_USER_DIR/storage/e/ed/1a/ed1aa60f0706cefde8ba2b3be662d3a0e0e1fbc94a52a3201944684cc0c5f244
+
+    if ! ghe-backup -v > "$TRASHDIR/backup-out" 2>&1; then
+      cat "$TRASHDIR/backup-out"
+      : ghe-backup should have completed successfully
+      false
+    fi
+
+    # Check the output for the warnings
+    grep -q "Warning: One or more repository networks and/or gists were not found on the source appliance." "$TRASHDIR/backup-out"
+    grep -q "\-1/23/bb/4c/gist" "$TRASHDIR/backup-out"
+    grep -q "\-1/nw/23/bb/4c/2345" "$TRASHDIR/backup-out"
+    grep -q "Warning: One or more storage objects were not found on the source appliance." "$TRASHDIR/backup-out"
+    grep -q "\-e/ed/1a/ed1aa60f0706cefde8ba2b3be662d3a0e0e1fbc94a52a3201944684cc0c5f244" "$TRASHDIR/backup-out"
+
+    verify_all_backedup_data
+)
+end_test

test/test-ghe-restore.sh

@@ -385,7 +385,7 @@ begin_test "ghe-restore cluster"
 
   # CI servers may have moreutils parallel and GNU parallel installed. We need moreutils parallel.
   if [ -x "/usr/bin/parallel.moreutils" ]; then
-    ln -s /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
+    ln -sf /usr/bin/parallel.moreutils "$ROOTDIR/test/bin/parallel"
   fi
 
   # run ghe-restore and write output to file for asserting against