Clean up stale HA nodes on restore

.travis.yml

@@ -7,6 +7,7 @@ matrix:
         - brew install gnu-tar
         - brew install moreutils
         - brew install shellcheck
+        - brew install jq
       script: make test
     - os: linux
       dist: trusty
@@ -21,4 +22,5 @@ matrix:
             - debhelper
             - moreutils
             - fakeroot
+            - jq
       script: debuild -uc -us

bin/ghe-restore

@@ -341,6 +341,22 @@ else
   fi
 fi
 
+# Clean up all stale replicas
+if ! $CLUSTER; then
+  restored_uuid=$(cat $GHE_RESTORE_SNAPSHOT_PATH/uuid)
+  other_nodes=$(echo "
+    set -o pipefail; \
+    ghe-spokes server show --json \
+    | jq -r '.[] | select(.host | contains(\"git-server\")).host' \
+    | sed 's/^git-server-//g' | grep -F -x -v \"$restored_uuid\"" \
+  | ghe-ssh "$GHE_HOSTNAME" -- /bin/bash)
+  if [ -n "$other_nodes" ]; then
+    echo "Cleaning up stale nodes ..."
+    for uuid in $other_nodes; do
+      ghe-ssh "$GHE_HOSTNAME" -- "/usr/local/share/enterprise/ghe-cluster-cleanup-node $uuid" 1>&3
+    done
+  fi
+fi
 
 # Update the remote status to "complete". This has to happen before importing
 # ssh host keys because subsequent commands will fail due to the host key

test/bin/ghe-cluster-cleanup-node

@@ -0,0 +1 @@
+ghe-fake-finalize-command

test/bin/ghe-spokes

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+# Usage: ghe-spokes
+# Emulates the remote GitHub ghe-spokes command. Tests use this
+# to assert that the command was executed.
+set -e
+echo '[
+  {
+    "host": "git-server-fake-uuid"
+  },
+  {
+    "host": "git-server-fake-uuid1"
+  },
+  {
+    "host": "git-server-fake-uuid2"
+  }
+]'

test/testlib.sh

@@ -413,6 +413,8 @@ verify_all_restored_data() {
     grep -q "fake ghe-export-ssh-host-keys data" "$TRASHDIR/restore-out"
     # verify all ES data was transferred from live directory to the temporary restore directory
     diff -ru --exclude="*.gz" "$GHE_DATA_DIR/current/elasticsearch" "$GHE_REMOTE_DATA_USER_DIR/elasticsearch-restore"
+    # verify stale servers were cleared
+    grep -q "ghe-cluster-cleanup-node OK" "$TRASHDIR/restore-out"
   else
     grep -q "fake audit log last yr last mth" "$TRASHDIR/restore-out"
     grep -q "fake audit log this yr this mth" "$TRASHDIR/restore-out"