Enable test reruns on failed fragiled tests

Added ability to set per suite in suite.ini configuration file
'retries' option, which sets the number of accepted reruns of
the tests failed from 'fragile' list:

  fragile = {
    "retries": 10,
    "tests": {
        "bitset.test.lua": {
            "issues": [ "gh-4095" ],
        }
    }}

Part of #189

dispatcher.py

@@ -365,6 +365,7 @@ def __init__(self, key, task_group, randomize):
                 random.shuffle(self.task_ids)
         else:
             self.randomize = False
+        self.is_fragile = task_group['is_fragile']
         self.result_queue = SimpleQueue()
         self.task_queue = SimpleQueue()
 
@@ -388,7 +389,7 @@ def _run_worker(self, worker_id, tcp_port_range):
         os.environ['TEST_RUN_TCP_PORT_END'] = str(tcp_port_range[1])
         color_stdout.queue = self.result_queue
         worker = self.gen_worker(worker_id)
-        worker.run_all(self.task_queue, self.result_queue)
+        worker.run_all(self.task_queue, self.result_queue, self.is_fragile)
 
     def add_worker(self, worker_id, tcp_port_range):
         # Note: each of our workers should consume only one None, but for the

lib/test_suite.py

@@ -81,7 +81,7 @@ def __init__(self, suite_path, args):
         self.args = args
         self.tests = []
         self.ini = {}
-        self.fragile = {'tests': {}}
+        self.fragile = {'retries': 0, 'tests': {}}
         self.suite_path = suite_path
         self.ini["core"] = "tarantool"
 
@@ -265,5 +265,8 @@ def run_test(self, test, server, inspector):
     def is_parallel(self):
         return self.ini['is_parallel']
 
+    def fragile_retries(self):
+        return self.fragile.get('retries', 0)
+
     def show_reproduce_content(self):
         return self.ini['show_reproduce_content']

lib/worker.py

@@ -82,13 +82,15 @@ def get_task_groups():
                 'gen_worker': gen_worker,
                 'task_ids': stable_task_ids,
                 'is_parallel': suite.is_parallel(),
+                'is_fragile': False,
                 'show_reproduce_content': suite.show_reproduce_content(),
             }
         if fragile_task_ids:
             res[key + '_fragile'] = {
                 'gen_worker': gen_worker,
                 'task_ids': fragile_task_ids,
                 'is_parallel': False,
+                'is_fragile': True,
                 'show_reproduce_content': suite.show_reproduce_content(),
             }
     return res
@@ -314,7 +316,7 @@ def run_task(self, task_id):
             raise
         return short_status
 
-    def run_loop(self, task_queue, result_queue):
+    def run_loop(self, task_queue, result_queue, is_fragile):
         """ called from 'run_all' """
         while True:
             task_id = self.task_get(task_queue)
@@ -326,8 +328,24 @@ def run_loop(self, task_queue, result_queue):
                 self.stop_worker(task_queue, result_queue)
                 break
 
+            short_status = None
             result_queue.put(self.current_task(task_id))
-            short_status = self.run_task(task_id)
+            if is_fragile:
+                retries_left = self.suite.fragile_retries()
+                # let's run till short_status became 'pass'
+                while short_status != 'pass' and retries_left >= 0:
+                    # print message only after some fails occurred
+                    if short_status == 'fail':
+                        color_stdout(
+                            'Test "%s", conf: "%s"\n'
+                            '\tfrom "fragile" list failed, rerunning ...\n'
+                            % (task_id[0], task_id[1]), schema='error')
+                    # run task and save the result to short_status
+                    short_status = self.run_task(task_id)
+                    retries_left = retries_left - 1
+            else:
+                short_status = self.run_task(task_id)
+
             result_queue.put(self.wrap_result(task_id, short_status))
             if not lib.Options().args.is_force and short_status == 'fail':
                 color_stdout(
@@ -341,14 +359,14 @@ def run_loop(self, task_queue, result_queue):
                 raise VoluntaryStopException()
             self.task_done(task_queue)
 
-    def run_all(self, task_queue, result_queue):
+    def run_all(self, task_queue, result_queue, is_fragile):
         if not self.initialized:
             self.flush_all_tasks(task_queue, result_queue)
             result_queue.put(self.done_marker())
             return
 
         try:
-            self.run_loop(task_queue, result_queue)
+            self.run_loop(task_queue, result_queue, is_fragile)
         except (KeyboardInterrupt, Exception) as e:
             if not isinstance(e, KeyboardInterrupt) and \
                not isinstance(e, VoluntaryStopException):