Make the test more robust. Part of this required modifying GetInState…

…, which is racy in that in between calling pollState and setting up event monitoring, the cluster could've elected a leader. When that happens, Leader() errors returning 0 leaders.

raft_test.go

@@ -2337,42 +2337,67 @@ func TestRaft_LeadershipTransferWithOneNode(t *testing.T) {
 	}
 }
 
-func TestRaft_LeadershipTransferWithSevenNodes(t *testing.T) {
+func TestRaft_LeadershipTransferWithWrites(t *testing.T) {
 	conf := inmemConfig(t)
 	conf.Logger = hclog.New(&hclog.LoggerOptions{Level: hclog.Trace})
 	c := MakeCluster(7, t, conf)
 	defer c.Close()
 
-	// Wait for a leader
-	leader := c.Leader()
-
 	doneCh := make(chan struct{})
-	t.Cleanup(func() {
-		close(doneCh)
-	})
+	var writerErr error
+	var wg sync.WaitGroup
+	var writes int
+	wg.Add(1)
+	leader := c.Leader()
 	go func() {
+		defer wg.Done()
 		for {
 			select {
 			case <-doneCh:
 				return
 			default:
 				future := leader.Apply([]byte("test"), 0)
-				if err := future.Error(); err != nil {
-					if errors.Is(err, ErrRaftShutdown) || errors.Is(err, ErrNotLeader) {
-						return
-					}
-					if !errors.Is(err, ErrLeadershipTransferInProgress) {
-						t.Logf("[ERR] err: %v", err)
-					}
+				switch err := future.Error(); {
+				case errors.Is(err, ErrRaftShutdown):
+					return
+				case errors.Is(err, ErrNotLeader):
+					leader = c.Leader()
+				case errors.Is(err, ErrLeadershipTransferInProgress):
+					continue
+				case errors.Is(err, ErrLeadershipLost):
+					continue
+				case err == nil:
+					writes++
+				default:
+					writerErr = err
 				}
-				time.Sleep(time.Microsecond)
+				time.Sleep(time.Millisecond)
 			}
 		}
 	}()
-	time.Sleep(time.Second / 2)
+
+	follower := c.Followers()[0]
+	future := c.Leader().LeadershipTransferToServer(follower.localID, follower.localAddr)
+	if future.Error() != nil {
+		t.Fatalf("Didn't expect error: %v", future.Error())
+	}
+	if follower.localID != c.Leader().localID {
+		t.Error("Leadership should have been transitioned to specified server.")
+	}
+	close(doneCh)
+	wg.Wait()
+	if writerErr != nil {
+		t.Fatal(writerErr)
+	}
+	t.Logf("writes: %d", writes)
+}
+
+func TestRaft_LeadershipTransferWithSevenNodes(t *testing.T) {
+	c := MakeCluster(7, t, nil)
+	defer c.Close()
 
 	follower := c.GetInState(Follower)[0]
-	future := leader.LeadershipTransferToServer(follower.localID, follower.localAddr)
+	future := c.Leader().LeadershipTransferToServer(follower.localID, follower.localAddr)
 	if future.Error() != nil {
 		t.Fatalf("Didn't expect error: %v", future.Error())
 	}

testing.go

@@ -433,7 +433,7 @@ func (c *cluster) GetInState(s RaftState) []*Raft {
 	// restart the timer.
 	pollStartTime := time.Now()
 	for {
-		inState, highestTerm := c.pollState(s)
+		_, highestTerm := c.pollState(s)
 		inStateTime := time.Now()
 
 		// Sometimes this routine is called very early on before the
@@ -479,8 +479,9 @@ func (c *cluster) GetInState(s RaftState) []*Raft {
 				c.t.Fatalf("timer channel errored")
 			}
 
-			c.logger.Info(fmt.Sprintf("stable state for %s reached at %s (%d nodes), %s from start of poll, %s from cluster start. Timeout at %s, %s after stability",
-				s, inStateTime, len(inState), inStateTime.Sub(pollStartTime), inStateTime.Sub(c.startTime), t, t.Sub(inStateTime)))
+			inState, highestTerm := c.pollState(s)
+			c.logger.Info(fmt.Sprintf("stable state for %s reached at %s (%d nodes), highestTerm is %d, %s from start of poll, %s from cluster start. Timeout at %s, %s after stability",
+				s, inStateTime, len(inState), highestTerm, inStateTime.Sub(pollStartTime), inStateTime.Sub(c.startTime), t, t.Sub(inStateTime)))
 			return inState
 		}
 	}