Permalink
Browse files

CBQE-2325: Implement stop graceful failover logic in tests

Change-Id: Icc0ea01960cdb37b0d85263152a0edcd51e6db9d
Reviewed-on: http://review.couchbase.org/39420
Tested-by: buildbot <build@couchbase.com>
Reviewed-by: Thuan Nguyen <soccon@gmail.com>
  • Loading branch information...
1 parent 1dd04eb commit f17d3c2cba34957f970647059ea1940c0f85d74f Parag Agarwal committed with paragagarwal Jul 15, 2014
Showing with 18 additions and 3 deletions.
  1. +1 −0 conf/py-newfailover.conf
  2. +1 −0 pytests/failover/failoverbasetests.py
  3. +16 −3 pytests/failover/failovertests.py
View
1 conf/py-newfailover.conf
@@ -30,6 +30,7 @@ failover.failovertests.FailoverTests:
# Graceful Failover and or Delta Recovery
test_failover_normal,replicas=1,num_failed_nodes=1,items=100000,dgm_run=True,,failoverMaster=True,graceful=True,GROUP=P0;GRACEFUL
+ test_failover_normal,replicas=1,num_failed_nodes=1,items=200000,vbuckets=1024,stopGracefulFailover=True,dgm_run=True,,failoverMaster=True,graceful=True,GROUP=P0;GRACEFUL
test_failover_normal,replicas=1,num_failed_nodes=1,items=100000,dgm_run=True,graceful=True,GROUP=P0;GRACEFUL
test_failover_normal,replicas=1,num_failed_nodes=1,load_ratio=20,sasl_buckets=1,graceful=True,GROUP=P1;GRACEFUL
test_failover_firewall,replicas=1,num_failed_nodes=1,gracefulFailoverFail=True,items=100000,graceful=True,GROUP=P1;GRACEFUL
View
1 pytests/failover/failoverbasetests.py
@@ -28,6 +28,7 @@ def setUp(self):
self.check_verify_failover_type = self.input.param("check_verify_failover_type", True)
self.recoveryType = self.input.param("recoveryType", "delta")
self.bidirectional = self.input.param("bidirectional", False)
+ self.stopGracefulFailover = self.input.param("stopGracefulFailover", False)
self._value_size = self.input.param("value_size", 256)
self.doc_ops = self.input.param("doc_ops", [])
self.deltaRecoveryBuckets = self.input.param("deltaRecoveryBuckets", None)
View
19 pytests/failover/failovertests.py
@@ -268,8 +268,21 @@ def run_failover_operations(self, chosen, failover_reason):
# define precondition check for failover
success_failed_over = self.rest.fail_over(node.id, graceful=(self.graceful and graceful_failover))
if self.graceful and graceful_failover:
- msg = "rebalance failed while removing failover nodes {0}".format(node.id)
- self.assertTrue(self.rest.monitorRebalance(stop_if_loop=True), msg=msg)
+ if self.stopGracefulFailover:
+ # Logic to stop graceful and start it again
+ # Stop Graceful
+ reached = RestHelper(self.rest).rebalance_reached(20)
+ if reached:
+ self.log.info(" Stopping Graceful failover")
+ stopped = self.rest.stop_rebalance(wait_timeout=self.wait_timeout / 3)
+ self.assertTrue(stopped, msg="unable to stop rebalance")
+ # Start Graceful Again
+ success_failed_over = self.rest.fail_over(node.id, graceful=(self.graceful and graceful_failover))
+ else:
+ self.assertTrue(False, msg="graceful failover finished before we could stop graceful failover, please check test scenario")
+ else:
+ msg = "rebalance failed while removing failover nodes {0}".format(node.id)
+ self.assertTrue(self.rest.monitorRebalance(stop_if_loop=True), msg=msg)
failed_over = failed_over and success_failed_over
# Check for negative cases
@@ -289,7 +302,7 @@ def run_failover_operations(self, chosen, failover_reason):
self.log.info("unable to failover the node the first time. try again in 60 seconds..")
# try again in 75 seconds
self.sleep(75)
- failed_over = self.rest.fail_over(node.id, graceful=self.graceful)
+ failed_over = self.rest.fail_over(node.id, graceful=(self.graceful and graceful_failover))
if self.graceful and (failover_reason not in ['stop_server', 'firewall']):
reached = RestHelper(self.rest).rebalance_reached()
self.assertTrue(reached, "rebalance failed for Graceful Failover, stuck or did not completed")

0 comments on commit f17d3c2

Please sign in to comment.