Upgrade raft-autopilot and wait for autopilot it to stop when revoking leadership

.changelog/9626.txt

@@ -0,0 +1,3 @@
+```release-note:bug
+autopilot: Fixed a bug that would cause snapshot restoration to stop autopilot on the leader.
+```

agent/consul/leader.go

@@ -395,7 +395,9 @@ func (s *Server) revokeLeadership() {
 
 	s.resetConsistentReadReady()
 
-	s.autopilot.Stop()
+	// Stop returns a chan and we want to block until it is closed
+	// which indicates that autopilot is actually stopped.
+	<-s.autopilot.Stop()
 }
 
 // DEPRECATED (ACL-Legacy-Compat) - Remove once old ACL compatibility is removed

agent/consul/snapshot_endpoint_test.go

@@ -13,6 +13,8 @@ import (
 	"github.com/hashicorp/consul/sdk/testutil/retry"
 	"github.com/hashicorp/consul/testrpc"
 	msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc"
+	autopilot "github.com/hashicorp/raft-autopilot"
+	"github.com/stretchr/testify/require"
 )
 
 // verifySnapshot is a helper that does a snapshot and restore.
@@ -165,6 +167,11 @@ func TestSnapshot(t *testing.T) {
 
 	testrpc.WaitForLeader(t, s1.RPC, "dc1")
 	verifySnapshot(t, s1, "dc1", "")
+
+	// ensure autopilot is still running
+	// https://github.com/hashicorp/consul/issues/9626
+	apstatus, _ := s1.autopilot.IsRunning()
+	require.Equal(t, autopilot.Running, apstatus)
 }
 
 func TestSnapshot_LeaderState(t *testing.T) {

go.mod

@@ -53,7 +53,7 @@ require (
 	github.com/hashicorp/memberlist v0.2.2
 	github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69
 	github.com/hashicorp/raft v1.2.0
-	github.com/hashicorp/raft-autopilot v0.1.1
+	github.com/hashicorp/raft-autopilot v0.1.2
 	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea
 	github.com/hashicorp/serf v0.9.5
 	github.com/hashicorp/vault/api v1.0.5-0.20200717191844-f687267c8086

go.sum

@@ -289,8 +289,8 @@ github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69/go.mo
 github.com/hashicorp/raft v1.1.1/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
 github.com/hashicorp/raft v1.2.0 h1:mHzHIrF0S91d3A7RPBvuqkgB4d/7oFJZyvf1Q4m7GA0=
 github.com/hashicorp/raft v1.2.0/go.mod h1:vPAJM8Asw6u8LxC3eJCUZmRP/E4QmUGE1R7g7k8sG/8=
-github.com/hashicorp/raft-autopilot v0.1.1 h1:f8Dv2y1Vq8ttuH2+oh5l87Paj/BINpMm5TBrMLx+qGQ=
-github.com/hashicorp/raft-autopilot v0.1.1/go.mod h1:HUBUSYtpQRVkgjvvoOgsZPvwe6b6FZJ1xXtaftRZvrA=
+github.com/hashicorp/raft-autopilot v0.1.2 h1:yeqdUjWLjVJkBM+mcVxqwxi+w+aHsb9cEON2dz69OCs=
+github.com/hashicorp/raft-autopilot v0.1.2/go.mod h1:Af4jZBwaNOI+tXfIqIdbcAnh/UyyqIMj/pOISIfhArw=
 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea h1:xykPFhrBAS2J0VBzVa5e80b5ZtYuNQtgXjN40qBZlD4=
 github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea/go.mod h1:pNv7Wc3ycL6F5oOWn+tPGo2gWD4a5X+yp/ntwdKLjRk=
 github.com/hashicorp/serf v0.9.5 h1:EBWvyu9tcRszt3Bxp3KNssBMP1KuHWyO51lz9+786iM=