Backport testrunner changes from 7.0.0 to 6.3.3

Change-Id: Ic40097bf23e2aefc9319627db48e8f5feb2b4c8d
Reviewed-on: http://review.couchbase.org/c/testrunner/+/156267
Tested-by: <evgeny.makarenko@couchbase.com>
Reviewed-by: Girish Benakappa <girish.benakappa@couchbase.com>

conf/fts/py-fts-upgrade.conf

@@ -98,4 +98,4 @@ upgrade.upgrade_tests.UpgradeTests:
      # create default index before upgrade,
      # update to upside_down during upgrade,
      # update to scorch after upgrade, run queries
-     test_upgrade,items=50000,initial_version=5.0.1-5003,nodes_init=3,initialize_events=create_fts_index_query_compare,initial-services-setting=kv,fts-kv,kv-fts,upgrade_services_in=same,in_between_events=update_index_to_upside_down-run_fts_query_and_compare,after_events=rebalance_in-update_index_to_scorch-run_fts_query_and_compare,after_upgrade_services_in=kv,fts,disable_HTP=True,upgrade_test=True,GROUP=MIXED_CLUSTER
+     test_upgrade,items=50000,initial_version=5.0.1-5003,nodes_init=3,initialize_events=create_fts_index_query_compare,initial-services-setting=kv,fts-kv,kv-fts,upgrade_services_in=same,in_between_events=update_index_to_upside_down-run_fts_query_and_compare,after_events=rebalance_in-update_index_to_scorch-run_fts_query_and_compare,after_upgrade_services_in=kv,fts,disable_HTP=True,upgrade_test=True,GROUP=MIXED_CLUSTER

lib/membase/api/rest_client.py

@@ -2141,7 +2141,18 @@ def get_dcp_queue_size(self, bucket):
     def get_active_key_count(self, bucket):
         """Fetch bucket stats and return the bucket's curr_items count"""
         bucket_stats = self.fetch_bucket_stats(bucket)
-        return bucket_stats['op']['samples']['curr_items'][-1]
+        ret_val = -1
+        retries = 10
+        while retries > 0:
+            try:
+                ret_val = bucket_stats['op']['samples']['curr_items'][-1]
+                return ret_val
+            except KeyError as err:
+                log.error("get_active_key_count() function for bucket {0} reported an error {1}".format(bucket, err))
+                log.error("Corresponding bucket stats JSON is {0}".format(bucket_stats))
+                time.sleep(2)
+                retries = retries - 1
+        return ret_val
 
     def get_replica_key_count(self, bucket):
         """Fetch bucket stats and return the bucket's replica count"""
@@ -3139,7 +3150,7 @@ def run_fts_query(self, index_name, query_json, timeout=70):
             return content['total_hits'], content['hits'], content['took'], \
                    content['status']
         else:
-            return -1, content, -1, status
+            return -1, content['error'], -1, content['status']
 
     def run_fts_query_with_facets(self, index_name, query_json):
         """Method run an FTS query through rest api"""

lib/remote/remote_util.py

@@ -569,6 +569,9 @@ def kill_erlang(self, os="unix", delay=0):
                     break
         else:
             o, r = self.execute_command("killall -9 beam.smp")
+            if r and r[0] and "command not found" in r[0]:
+                o, r = self.execute_command("pkill beam.smp")
+                self.log_command_output(o, r)
             self.log_command_output(o, r, debug=False)
             all_killed = False
             count = 0
@@ -583,6 +586,9 @@ def kill_erlang(self, os="unix", delay=0):
                     all_killed = True
                 if count == 3:
                     o, r = self.execute_command("killall -9 beam.smp")
+                    if r and r[0] and "command not found" in r[0]:
+                        o, r = self.execute_command("pkill beam.smp")
+                        self.log_command_output(o, r)
                 count += 1
             if not all_killed:
                 raise Exception("Could not kill erlang process")
@@ -595,6 +601,9 @@ def kill_cbft_process(self):
             self.log_command_output(o, r)
         else:
             o, r = self.execute_command("killall -9 cbft")
+            if r and r[0] and "command not found" in r[0]:
+                o, r = self.execute_command("pkill cbft")
+                self.log_command_output(o, r)
             self.log_command_output(o, r)
         return o, r
 
@@ -836,6 +845,7 @@ def stop_moxi(self):
             self.log_command_output(o, r)
         else:
             raise Exception("stopping standalone moxi is not supported on windows")
+
     def is_url_live(self, url, exit_if_not_live=True):
         live_url = False
         #log.info("Check if url {0} is ok".format(url))

pytests/newupgradebasetest.py

@@ -994,6 +994,7 @@ def update_delete_fts_data_run_queries(self, fts_obj):
         :param fts_obj: the FTS object created in create_fts_index_query_compare()
         """
         fts_obj.async_perform_update_delete()
+        self.fts_obj.wait_for_indexing_complete()
         for index in fts_obj.fts_indexes:
             fts_obj.run_query_and_compare(index)
 

pytests/upgrade/newupgradebasetest.py

@@ -1043,6 +1043,7 @@ def update_delete_fts_data_run_queries(self, queue=None):
         :param fts_obj: the FTS object created in create_fts_index_query_compare()
         """
         self.fts_obj.async_perform_update_delete()
+        self.fts_obj.wait_for_indexing_complete()
         for index in self.fts_obj.fts_indexes:
             self.fts_obj.run_query_and_compare(index)
 
@@ -1097,6 +1098,7 @@ def modify_num_pindexes(self, queue=None):
     def modify_num_replicas(self, queue=None):
         for index in self.fts_obj.fts_indexes:
             index.update_num_replicas(1)
+        self.fts_obj.wait_for_indexing_complete()
 
     def update_index_to_scorch(self, queue=None):
         try: