FAB-15210 fix Raft UT flake

Recall that Raft reconfiguration is done with two rounds of consensus:
1) Config Block first
2) then Raft config entry

To precisely mimic leader failover between 1) and 2) in Raft UT, we
monitor number of Raft message being disseminated by leader, and
disconnect network at certain point of time.

However, in some cases, leader might optimistically replicate Raft
config entry along with the 2nd round of MsgApp for Config block
(recall 2 phase commit in Raft).

In such case, we need to pick the follower with higher index to be
elected as new leader, otherwise a node may never be able to claim
leadership and the test would fail.

In addition, this CR also reduces LeaderCheckInterval to 500ms to
speed up test run. It was set to 10s by default.

Change-Id: I7e735d5280928152420c174bfd2b1d54cbae4f25
Signed-off-by: Jay Guo <guojiannan1101@gmail.com>

Author: guoger

orderer/consensus/etcdraft/chain_test.go

@@ -2302,7 +2302,16 @@ var _ = Describe("Chain", func() {
 							Eventually(c.support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
 						})
 					c1.setStepFunc(step1)
-					network.elect(2)
+
+					// elect node with higher index
+					i2, _ := c2.storage.LastIndex() // err is always nil
+					i3, _ := c3.storage.LastIndex()
+					candidate := uint64(2)
+					if i3 > i2 {
+						candidate = 3
+					}
+					network.chains[candidate].cutter.CutNext = true
+					network.elect(candidate)
 
 					_, raftmetabytes := c1.support.WriteConfigBlockArgsForCall(0)
 					meta := &common.Metadata{Value: raftmetabytes}
@@ -2325,7 +2334,6 @@ var _ = Describe("Chain", func() {
 					Eventually(c4.support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
 
 					By("submitting new transaction to follower")
-					c2.cutter.CutNext = true
 					err = c4.Order(env, 0)
 					Expect(err).NotTo(HaveOccurred())
 
@@ -2337,12 +2345,7 @@ var _ = Describe("Chain", func() {
 					// node 1 has been stopped should not write any block
 					Consistently(c1.support.WriteBlockCallCount).Should(Equal(1))
 
-					network.connect(1)
-
-					c2.clock.Increment(interval)
-					// check that former leader didn't get stuck and actually got resign signal,
-					// and once connected capable of communicating with rest of the replicas set
-					Eventually(c1.observe, LongEventualTimeout).Should(Receive(Equal(raft.SoftState{Lead: 2, RaftState: raft.StateFollower})))
+					network.join(1, true)
 					Eventually(c1.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
 				})
 
@@ -2400,8 +2403,15 @@ var _ = Describe("Chain", func() {
 						network.connect(i)
 					}
 
-					By("re-elect node 2 to be a leader")
-					network.elect(2)
+					// elect node with higher index
+					i2, _ := c2.storage.LastIndex() // err is always nil
+					i3, _ := c3.storage.LastIndex()
+					candidate := uint64(2)
+					if i3 > i2 {
+						candidate = 3
+					}
+					network.chains[candidate].cutter.CutNext = true
+					network.elect(candidate)
 
 					c4.start()
 					Expect(c4.WaitReady()).To(Succeed())
@@ -2411,12 +2421,9 @@ var _ = Describe("Chain", func() {
 					Eventually(c4.support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
 
 					By("submitting new transaction to follower")
-					c2.cutter.CutNext = true
 					err = c4.Order(env, 0)
 					Expect(err).NotTo(HaveOccurred())
 
-					c2.clock.Increment(interval)
-
 					// rest nodes are alive include a newly added, hence should write 2 blocks
 					Eventually(c1.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
 					Eventually(c2.support.WriteBlockCallCount, LongEventualTimeout).Should(Equal(2))
@@ -2456,11 +2463,17 @@ var _ = Describe("Chain", func() {
 						Eventually(c.support.WriteConfigBlockCallCount, LongEventualTimeout).Should(Equal(1))
 					})
 
-					// electing new leader
-					network.elect(2)
+					// elect node with higher index
+					i2, _ := c2.storage.LastIndex() // err is always nil
+					i3, _ := c3.storage.LastIndex()
+					candidate := uint64(2)
+					if i3 > i2 {
+						candidate = 3
+					}
+					network.chains[candidate].cutter.CutNext = true
+					network.elect(candidate)
 
 					By("submitting new transaction to follower")
-					c2.cutter.CutNext = true
 					err = c3.Order(env, 0)
 					Expect(err).NotTo(HaveOccurred())
 
@@ -3459,20 +3472,21 @@ func newChain(timeout time.Duration, channel string, dataDir string, id uint64,
 	fakeFields := newFakeMetricsFields()
 
 	opts := etcdraft.Options{
-		RaftID:            uint64(id),
-		Clock:             clock,
-		TickInterval:      interval,
-		ElectionTick:      ELECTION_TICK,
-		HeartbeatTick:     HEARTBEAT_TICK,
-		MaxSizePerMsg:     1024 * 1024,
-		MaxInflightBlocks: 256,
-		BlockMetadata:     raftMetadata,
-		Consenters:        consenters,
-		Logger:            flogging.NewFabricLogger(zap.NewExample()),
-		MemoryStorage:     storage,
-		WALDir:            path.Join(dataDir, "wal"),
-		SnapDir:           path.Join(dataDir, "snapshot"),
-		Metrics:           newFakeMetrics(fakeFields),
+		RaftID:              uint64(id),
+		Clock:               clock,
+		TickInterval:        interval,
+		ElectionTick:        ELECTION_TICK,
+		HeartbeatTick:       HEARTBEAT_TICK,
+		MaxSizePerMsg:       1024 * 1024,
+		MaxInflightBlocks:   256,
+		BlockMetadata:       raftMetadata,
+		LeaderCheckInterval: 500 * time.Millisecond,
+		Consenters:          consenters,
+		Logger:              flogging.NewFabricLogger(zap.NewExample()),
+		MemoryStorage:       storage,
+		WALDir:              path.Join(dataDir, "wal"),
+		SnapDir:             path.Join(dataDir, "snapshot"),
+		Metrics:             newFakeMetrics(fakeFields),
 	}
 
 	support := &consensusmocks.FakeConsenterSupport{}
@@ -3901,6 +3915,7 @@ func (n *network) elect(id uint64) {
 	n.RUnlock()
 
 	// Send node an artificial MsgTimeoutNow to emulate leadership transfer.
+	fmt.Fprintf(GinkgoWriter, "Send artificial MsgTimeoutNow to elect node %d\n", id)
 	candidate.Consensus(&orderer.ConsensusRequest{Payload: protoutil.MarshalOrPanic(&raftpb.Message{Type: raftpb.MsgTimeoutNow})}, 0)
 	Eventually(candidate.observe, LongEventualTimeout).Should(Receive(StateEqual(id, raft.StateLeader)))