From 74111dd8d95fbbcf318d4d1b49822b0cb0af20db Mon Sep 17 00:00:00 2001 From: Kevin Cherkauer Date: Mon, 21 Dec 2020 15:25:59 -0800 Subject: [PATCH] MB-43391 Add sleep-retry loop to framework ResetCluster() Replace hard-coded 10-sec sleep in test framework's ResetCluster() function with a sleep-retry loop that will try immediately and then every 1 second thereafter for up to 30 seconds before it fails. Interim attempts do not log any errors; an error is logged only if the final try still fails. Change-Id: I6d8ab9c0790d1007e8a8e922118f4ec18f2022e5 --- .../framework/clusterutility/cluster_setup.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/secondary/tests/framework/clusterutility/cluster_setup.go b/secondary/tests/framework/clusterutility/cluster_setup.go index e08c729a3..0a2e82738 100644 --- a/secondary/tests/framework/clusterutility/cluster_setup.go +++ b/secondary/tests/framework/clusterutility/cluster_setup.go @@ -262,13 +262,19 @@ func ResetCluster(serverAddr, username, password string, dropNodes []string, kee if err := waitForRebalanceFinish(serverAddr, username, password); err != nil { return fmt.Errorf("Error in resetCluster, err: %v", err) } - // Sleep for 10 seconds between rebalance so that couchbase-server gets sufficent - // time to go down and come back online - time.Sleep(10 * time.Second) for node, role := range keepNodes { - if err := AddNode(serverAddr, username, password, node, role); err != nil { - return fmt.Errorf("Error while adding node: %v (role: %v) to cluster, err: %v", node, role, err) + for retries := 0; ; retries++ { + err := AddNode(serverAddr, username, password, node, role) + if err == nil { + break + } + // Retries allow time for the node to come back up without a mandatory long sleep. + // Plain 10-second sleep was not always long enough. + if retries >= 30 { + return fmt.Errorf("Error while adding node: %v (role: %v) to cluster, err: %v", node, role, err) + } + time.Sleep(1 * time.Second) } } return nil