[FAB-12065] fix TestLeaderYield flaky test

Change-Id: Iaba09fe4902d421310f21378b0d8dbfb5e2ed107
Signed-off-by: Artem Barger <bartem@il.ibm.com>
(cherry picked from commit e9c2f4c)

core/deliverservice/deliveryclient.go

@@ -219,7 +219,7 @@ func (d *deliverServiceImpl) newClient(chainID string, ledgerInfoProvider blocks
 		return requester.RequestBlocks(ledgerInfoProvider)
 	}
 	backoffPolicy := func(attemptNum int, elapsedTime time.Duration) (time.Duration, bool) {
-		if elapsedTime > reconnectTotalTimeThreshold {
+		if elapsedTime >= reconnectTotalTimeThreshold {
 			return 0, false
 		}
 		sleepIncrement := float64(time.Millisecond * 500)

gossip/service/gossip_service_test.go

@@ -119,7 +119,7 @@ func TestLeaderElectionWithDeliverClient(t *testing.T) {
 	util.SetVal("peer.gossip.useLeaderElection", true)
 	util.SetVal("peer.gossip.orgLeader", false)
 	n := 10
-	gossips := startPeers(t, n, 20100)
+	gossips := startPeers(t, n, 20100, 0, 1, 2, 3, 4)
 
 	channelName := "chanA"
 	peerIndexes := make([]int, n)
@@ -178,7 +178,7 @@ func TestWithStaticDeliverClientLeader(t *testing.T) {
 	util.SetVal("peer.gossip.orgLeader", true)
 
 	n := 2
-	gossips := startPeers(t, n, 20200)
+	gossips := startPeers(t, n, 20200, 0, 1)
 
 	channelName := "chanA"
 	peerIndexes := make([]int, n)
@@ -232,7 +232,7 @@ func TestWithStaticDeliverClientNotLeader(t *testing.T) {
 	util.SetVal("peer.gossip.orgLeader", false)
 
 	n := 2
-	gossips := startPeers(t, n, 20300)
+	gossips := startPeers(t, n, 20300, 0, 1)
 
 	channelName := "chanA"
 	peerIndexes := make([]int, n)
@@ -272,7 +272,7 @@ func TestWithStaticDeliverClientBothStaticAndLeaderElection(t *testing.T) {
 	util.SetVal("peer.gossip.orgLeader", true)
 
 	n := 2
-	gossips := startPeers(t, n, 20400)
+	gossips := startPeers(t, n, 20400, 0, 1)
 
 	channelName := "chanA"
 	peerIndexes := make([]int, n)
@@ -297,7 +297,7 @@ func TestWithStaticDeliverClientBothStaticAndLeaderElection(t *testing.T) {
 				Committer: &mockLedgerInfo{1},
 				Store:     &mockTransientStore{},
 			})
-		}, "Dynamic leader lection based and static connection to ordering service can't exist simultaniosly")
+		}, "Dynamic leader election based and static connection to ordering service can't exist simultaneously")
 	}
 
 	stopPeers(gossips)
@@ -390,7 +390,7 @@ func TestLeaderElectionWithRealGossip(t *testing.T) {
 
 	// Creating gossip service instances for peers
 	n := 10
-	gossips := startPeers(t, n, 20500)
+	gossips := startPeers(t, n, 20500, 0, 1, 2, 3, 4)
 
 	// Joining all peers to first channel
 	channelName := "chanA"
@@ -629,15 +629,15 @@ func addPeersToChannel(t *testing.T, n int, portPrefix int, channel string, peer
 	waitUntilOrFailBlocking(t, wg.Wait, time.Second*10)
 }
 
-func startPeers(t *testing.T, n int, portPrefix int) []GossipService {
+func startPeers(t *testing.T, n int, portPrefix int, boot ...int) []GossipService {
 
 	peers := make([]GossipService, n)
 	wg := sync.WaitGroup{}
 	for i := 0; i < n; i++ {
 		wg.Add(1)
 		go func(i int) {
 
-			peers[i] = newGossipInstance(portPrefix, i, 100, 0, 1, 2, 3, 4, 5)
+			peers[i] = newGossipInstance(portPrefix, i, 100, boot...)
 			wg.Done()
 		}(i)
 	}

gossip/service/integration_test.go

@@ -105,19 +105,29 @@ func TestLeaderYield(t *testing.T) {
 	// Make sure the other peer declares itself as the leader soon after.
 	takeOverMaxTimeout := time.Minute
 	viper.Set("peer.gossip.election.leaderAliveThreshold", time.Second*5)
-	viper.Set("peer.deliveryclient.reconnectTotalTimeThreshold", time.Second*5)
+	// Test case has only two instance + making assertions only after membership view
+	// is stable, hence election duration could be shorter
+	viper.Set("peer.gossip.election.leaderElectionDuration", time.Millisecond*500)
+	// It's enough to make single re-try
+	viper.Set("peer.deliveryclient.reconnectTotalTimeThreshold", time.Second*1)
+	// Since we ensuring gossip has stable membership, there is no need for
+	// leader election to wait for stabilization
+	viper.Set("peer.gossip.election.membershipSampleInterval", time.Millisecond*100)
+	// There is no ordering service available anyway, hence connection timeout
+	// could be shorter
+	viper.Set("peer.deliveryclient.connTimeout", time.Millisecond*100)
 	viper.Set("peer.gossip.useLeaderElection", true)
 	viper.Set("peer.gossip.orgLeader", false)
 	n := 2
 	portPrefix := 30000
-	gossips := startPeers(t, n, portPrefix)
+	gossips := startPeers(t, n, portPrefix, 0, 1)
 	defer stopPeers(gossips)
 	channelName := "channelA"
 	peerIndexes := []int{0, 1}
 	// Add peers to the channel
 	addPeersToChannel(t, n, portPrefix, channelName, gossips, peerIndexes)
 	// Prime the membership view of the peers
-	waitForFullMembership(t, gossips, n, time.Second*30, time.Second*2)
+	waitForFullMembership(t, gossips, n, time.Second*30, time.Millisecond*100)
 	// Helper function that creates a gossipService instance
 	newGossipService := func(i int) *gossipServiceImpl {
 		gs := gossips[i].(*gossipServiceImpl)
@@ -164,12 +174,13 @@ func TestLeaderYield(t *testing.T) {
 	// Wait for p1 to take over. It should take over before time reaches timeLimit
 	timeLimit := time.Now().Add(takeOverMaxTimeout)
 	for getLeader() != 1 && time.Now().Before(timeLimit) {
-		time.Sleep(time.Second)
+		time.Sleep(100 * time.Millisecond)
 	}
 	if time.Now().After(timeLimit) {
 		util.PrintStackTrace()
 		t.Fatalf("p1 hasn't taken over leadership within %v: %d", takeOverMaxTimeout, getLeader())
 	}
+	t.Log("p1 has taken over leadership")
 	p0.chains[channelName].Stop()
 	p1.chains[channelName].Stop()
 	p0.deliveryService[channelName].Stop()