Skip to content

Commit

Permalink
[FAB-14217] Harden etcdraft eviction integration test
Browse files Browse the repository at this point in the history
The test finds a leader, and then creates a channel,
and then removes the leader from both channels.

However, obviously - that is only the system channel leader,
and it may not be the leader of the application channel.

When it's not the leader - the other nodes might cease communication
with that evicted node (in case it's slow) - and then the test fails
because it doesn't detect its own eviction.

This change set makes the test only evict the leader from the system channel,
and removes the creation of the application channel.

Change-Id: I8ff384c0c2003d137dccf9eba948d25bcc14188e
Signed-off-by: yacovm <yacovm@il.ibm.com>
  • Loading branch information
yacovm committed Feb 16, 2019
1 parent 8762acd commit 46ea35a
Showing 1 changed file with 10 additions and 19 deletions.
29 changes: 10 additions & 19 deletions integration/e2e/etcdraft_reconfig_test.go
Expand Up @@ -377,31 +377,22 @@ var _ = Describe("EndToEnd reconfiguration and onboarding", func() {
By("Waiting for them to elect a leader")
evictedNode := findLeader(ordererRunners) - 1

By("Creating a channel")
network.CreateChannel("testchannel", network.Orderers[evictedNode], peer)
By("Removing the leader from system channel")
serverCertBytes, err := ioutil.ReadFile(filepath.Join(network.OrdererLocalTLSDir(network.Orderers[evictedNode]), "server.crt"))
Expect(err).To(Not(HaveOccurred()))

By("Waiting for the channel to be serviced")
assertBlockReception(map[string]int{
"testchannel": 0,
}, orderers, peer, network)
nwo.RemoveConsenter(network, peer, network.Orderers[(evictedNode+1)%3], "systemchannel", serverCertBytes)

By("Removing the leader from both system channel and application channel")
certificatesOfOrderers := refreshOrdererPEMs(network)
for _, channelName := range []string{"systemchannel", "testchannel"} {
nwo.RemoveConsenter(network, peer, network.Orderers[(evictedNode+1)%3], channelName, certificatesOfOrderers[evictedNode].oldCert)
fmt.Fprintln(GinkgoWriter, "Ensuring the other orderers detect the eviction of the node on channel", "systemchannel")
Eventually(ordererRunners[(evictedNode+1)%3].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Deactivated node"))
Eventually(ordererRunners[(evictedNode+2)%3].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say("Deactivated node"))

fmt.Fprintln(GinkgoWriter, "Ensuring the other orderers detect the eviction of the node on channel", channelName)
Eventually(ordererRunners[(evictedNode+1)%3].Err(), time.Minute, time.Second).Should(gbytes.Say("Deactivated node"))
Eventually(ordererRunners[(evictedNode+2)%3].Err(), time.Minute, time.Second).Should(gbytes.Say("Deactivated node"))

fmt.Fprintln(GinkgoWriter, "Ensuring the evicted orderer stops rafting on channel", channelName)
stopMSg := fmt.Sprintf("Raft node stopped channel=%s", channelName)
Eventually(ordererRunners[evictedNode].Err(), time.Minute, time.Second).Should(gbytes.Say(stopMSg))
}
fmt.Fprintln(GinkgoWriter, "Ensuring the evicted orderer stops rafting on channel", "systemchannel")
stopMSg := fmt.Sprintf("Raft node stopped channel=%s", "systemchannel")
Eventually(ordererRunners[evictedNode].Err(), network.EventuallyTimeout, time.Second).Should(gbytes.Say(stopMSg))

By("Ensuring the evicted orderer now doesn't serve clients")
ensureEvicted(orderers[evictedNode], peer, network, "systemchannel")
ensureEvicted(orderers[evictedNode], peer, network, "testchannel")

By("Ensuring that all orderers don't log errors to the log")
assertNoErrorsAreLogged(ordererRunners)
Expand Down

0 comments on commit 46ea35a

Please sign in to comment.