CBQE-0:: fix hanging rebalance-out tests

Change-Id: Ib09ee90fbcd60f9a047b2fe41548fe694bd0d7f0
Reviewed-on: http://review.couchbase.org/41215
Reviewed-by: Parag Agarwal <agarwal.parag@gmail.com>
Tested-by: Parag Agarwal <agarwal.parag@gmail.com>

conf/rebalance/py-rebalanceinout.conf

@@ -5,8 +5,8 @@ rebalance.rebalanceinout.RebalanceInOutTests:
     rebalance_in_out_with_failover,nodes_in=1,nodes_init=3,replicas=1,items=10000,GROUP=IN_OUT;P0
     incremental_rebalance_in_out_with_mutation,items=0,replicas=2,GROUP=IN_OUT;P1
     incremental_rebalance_in_out_with_mutation,items=50000,value_size=512,replicas=3,GROUP=IN_OUT;P2
-    #incremental_rebalance_in_out_with_mutation_and_deletion,items=500000,value_size=256,max_verify=100000,GROUP=IN_OUT;P0
-    #incremental_rebalance_in_out_with_mutation_and_expiration,items=500000,value_size=512,max_verify=100000,GROUP=IN_OUT;P0
+    incremental_rebalance_in_out_with_mutation_and_deletion,items=100000,value_size=256,max_verify=100000,GROUP=IN_OUT;P0
+    incremental_rebalance_in_out_with_mutation_and_expiration,items=100000,value_size=512,max_verify=100000,GROUP=IN_OUT;P0
     incremental_rebalance_out_in_with_mutation,replicas=2,value_size=2048,GROUP=IN_OUT;P1
     incremental_rebalance_out_in_with_mutation,init_num_nodes=3,items=400000,GROUP=IN_OUT;P0
     incremental_rebalance_out_in_with_mutation,init_num_nodes=3,items=400000,active_resident_threshold=60,dgm_run=True,GROUP=IN_OUT;P0

conf/rebalance/py-rebalanceout.conf

@@ -36,7 +36,7 @@ rebalance.rebalanceout.RebalanceOutTests:
     incremental_rebalance_out_with_queries,blob_generator=False,items=100000,is_dev_ddoc=False,GROUP=OUT;P1;FROM_2_0
     rebalance_out_with_ddoc_compaction,items=500000,max_verify=100000,fragmentation_value=60,GROUP=OUT;BASIC;COMPACTION;P0;FROM_2_0
     #rebalance_out_with_ddoc_compaction,items=800000,nodes_in=3,replicas=2,fragmentation_value=85,max_verify=100000,GROUP=OUT;COMPACTION;P1;FROM_2_0
-    #incremental_rebalance_out_with_mutation_and_deletion,items=500000,value_size=256,max_verify=100000,GROUP=OUT;P0
+    incremental_rebalance_out_with_mutation_and_deletion,items=500000,value_size=256,max_verify=100000,GROUP=OUT;P0
     incremental_rebalance_out_with_mutation_and_expiration,items=500000,value_size=512,max_verify=100000,GROUP=OUT;P0
     rebalance_out_with_warming_up,items=1000000,max_verify=100000,max_verify=100000,GROUP=OUT;P1,BUGS=MB-7660_curr_items_tot_vb_replica_curr_items_mismatch_mixed_cluster_with_warming_up
     rebalance_out_with_warming_up,nodes_out=3,items=500000,replicas=1,max_verify=100000,value_size=1024,GROUP=OUT;P0,BUGS=MB-7660_curr_items_tot_vb_replica_curr_items_mismatch_mixed_cluster_with_warming_up

pytests/rebalance/rebalanceinout.py

@@ -389,7 +389,7 @@ def incremental_rebalance_out_in_with_mutation(self):
     def incremental_rebalance_in_out_with_mutation_and_deletion(self):
         self.cluster.rebalance(self.servers[:self.num_servers],
                                self.servers[1:self.num_servers], [])
-        gen_delete = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2,
+        gen_delete = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2 + 2000,
                               end=self.num_items)
         for i in reversed(range(self.num_servers)[self.num_servers / 2:]):
             tasks = self._async_load_all_buckets(self.master, self.gen_update, "update", 0,
@@ -420,12 +420,12 @@ def incremental_rebalance_in_out_with_mutation_and_deletion(self):
     def incremental_rebalance_in_out_with_mutation_and_expiration(self):
         self.cluster.rebalance(self.servers[:self.num_servers],
                                self.servers[1:self.num_servers], [])
-        gen_expire = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2,
+        gen_expire = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2 + 2000,
                               end=self.num_items)
         for i in reversed(range(self.num_servers)[self.num_servers / 2:]):
             self.log.info("iteration #{0}".format(i))
-            tasks = self._async_load_all_buckets(self.master, self.gen_update, "update", 0, batch_size=50)
-            tasks.extend(self._async_load_all_buckets(self.master, gen_expire, "update", 10, batch_size=50))
+            tasks = self._async_load_all_buckets(self.master, self.gen_update, "update", 0, batch_size=1000)
+            tasks.extend(self._async_load_all_buckets(self.master, gen_expire, "update", 10, batch_size=1000))
 
             self.cluster.rebalance(self.servers[:i], [], self.servers[i:self.num_servers])
             self.sleep(20)

pytests/rebalance/rebalanceout.py

@@ -562,13 +562,11 @@ def rebalance_out_with_ddoc_compaction(self):
     verifies that there has been no data loss, sum(curr_items) match the curr_items_total.
     Once all nodes have been rebalanced out of the cluster the test finishes."""
     def incremental_rebalance_out_with_mutation_and_deletion(self):
-        gen_2 = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2,
+        gen_2 = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2 + 2000,
                               end=self.num_items)
-        batch_size = 50
+        batch_size = 1000
         for i in reversed(range(self.num_servers)[1:]):
             # don't use batch for rebalance out 2-1 nodes
-            if i == 1:
-                batch_size = 1
             tasks = [self.cluster.async_rebalance(self.servers[:i], [], [self.servers[i]])]
             tasks += self._async_load_all_buckets(self.master, self.gen_update, "update", 0, batch_size=batch_size, timeout_secs=60)
             tasks += self._async_load_all_buckets(self.master, gen_2, "delete", 0, batch_size=batch_size, timeout_secs=60)
@@ -589,13 +587,11 @@ def incremental_rebalance_out_with_mutation_and_deletion(self):
     verifies that there has been no data loss, sum(curr_items) match the curr_items_total.
     Once all nodes have been rebalanced out of the cluster the test finishes."""
     def incremental_rebalance_out_with_mutation_and_expiration(self):
-        gen_2 = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2,
+        gen_2 = BlobGenerator('mike', 'mike-', self.value_size, start=self.num_items / 2 + 2000,
                               end=self.num_items)
-        batch_size = 50
+        batch_size = 1000
         for i in reversed(range(self.num_servers)[2:]):
             # don't use batch for rebalance out 2-1 nodes
-            if i == 1:
-                batch_size = 1
             rebalance = self.cluster.async_rebalance(self.servers[:i], [], [self.servers[i]])
             self._load_all_buckets(self.master, self.gen_update, "update", 0, batch_size=batch_size, timeout_secs=60)
             self._load_all_buckets(self.master, gen_2, "update", 5, batch_size=batch_size, timeout_secs=60)