From 0a0c6d8f7586cd9f2917e08b4fecb761ea66dd8c Mon Sep 17 00:00:00 2001 From: "Alexander V. Tikhonov" Date: Wed, 15 Jul 2020 09:34:35 +0300 Subject: [PATCH] Add ability to check failed tests w/ fragile list Added ability to check failed tests w/ fragile list to be sure that the current fail equal to the issue mentioned in the fragile list. Part of issue https://github.com/tarantool/tarantool/issues/5050 --- dispatcher.py | 4 +++- lib/test.py | 6 +++++- lib/test_suite.py | 8 ++++++++ lib/worker.py | 19 ++++++++++++------- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/dispatcher.py b/dispatcher.py index af8ba31b..df368d12 100644 --- a/dispatcher.py +++ b/dispatcher.py @@ -356,6 +356,7 @@ def __init__(self, key, task_group, randomize): self.task_ids = task_group['task_ids'] self.is_parallel = task_group['is_parallel'] self.fragile_retries = int(task_group['fragile_retries']) + self.fragile_checksums = task_group['fragile_checksums'] if self.is_parallel: self.randomize = randomize if self.randomize: @@ -385,7 +386,8 @@ def _run_worker(self, worker_id, tcp_port_range): os.environ['TEST_RUN_TCP_PORT_END'] = str(tcp_port_range[1]) color_stdout.queue = self.result_queue worker = self.gen_worker(worker_id) - worker.run_all(self.task_queue, self.result_queue, self.fragile_retries) + worker.run_all(self.task_queue, self.result_queue, self.fragile_retries, + self.fragile_checksums) def add_worker(self, worker_id, tcp_port_range): # Note: each of our workers should consume only one None, but for the diff --git a/lib/test.py b/lib/test.py index c607f225..106a9520 100644 --- a/lib/test.py +++ b/lib/test.py @@ -8,6 +8,7 @@ import sys import traceback from functools import partial +from hashlib import md5 try: from cStringIO import StringIO @@ -250,9 +251,12 @@ def run(self, server): color_stdout("[ updated ]\n", schema='test_new') else: has_result = os.path.exists(self.tmp_result) + result_checksum = '' if has_result: shutil.copy(self.tmp_result, self.reject) - short_status = 'fail' + with open(self.tmp_result, mode='rb') as result_file: + result_checksum = md5(result_file.read()).hexdigest() + short_status = 'fail:' + result_checksum color_stdout("[ fail ]\n", schema='test_fail') where = "" diff --git a/lib/test_suite.py b/lib/test_suite.py index f719febc..2487870a 100644 --- a/lib/test_suite.py +++ b/lib/test_suite.py @@ -174,6 +174,14 @@ def fragile_tests(self): res.append(test) return res + def fragile_checksums(self): + res = [] + for fragile in self.ini['fragile']: + checksum = re.split(r'md5sum:', fragile) + if not checksum[0]: + res.append(checksum[1]) + return res + def gen_server(self): try: return Server(self.ini, test_suite=self) diff --git a/lib/worker.py b/lib/worker.py index b41e2b2e..6e36625e 100644 --- a/lib/worker.py +++ b/lib/worker.py @@ -2,6 +2,7 @@ import copy import functools import os +import re import signal import traceback import yaml @@ -83,6 +84,7 @@ def get_task_groups(): 'task_ids': stable_task_ids, 'is_parallel': suite.is_parallel(), 'fragile_retries': 0, + 'fragile_checksums': suite.fragile_checksums(), 'show_reproduce_content': suite.show_reproduce_content(), } if fragile_task_ids: @@ -91,6 +93,7 @@ def get_task_groups(): 'task_ids': fragile_task_ids, 'is_parallel': False, 'fragile_retries': suite.fragile_retries(), + 'fragile_checksums': suite.fragile_checksums(), 'show_reproduce_content': suite.show_reproduce_content(), } return res @@ -315,7 +318,7 @@ def run_task(self, task_id): raise return short_status - def run_loop(self, task_queue, result_queue, fragile_retries): + def run_loop(self, task_queue, result_queue, fragile_retries, fragile_checksums): """ called from 'run_all' """ while True: task_id = self.task_get(task_queue) @@ -332,15 +335,17 @@ def run_loop(self, task_queue, result_queue, fragile_retries): # 'fragile_retries' value comes from the suite configuration retries_left = fragile_retries while short_status != 'pass' and retries_left >= 0: - if short_status == 'fail': + if re.match(r'^fail:.*', short_status): color_stdout( 'Test "%s", conf: "%s"\n' - '\tfrom "fragile" list failed - rerunning ...\n' - % (task_id[0], task_id[1]) , schema='error') + '\tfrom "fragile" list failed, result file checksum "%s" rerunning ...\n' + % (task_id[0], task_id[1], short_status), schema='error') short_status = self.run_task(task_id) + fail_checksum = re.split(r'fail:', short_status) + if fragile_checksums and not fail_checksum[1] in fragile_checksums: + break retries_left = retries_left - 1 - result_queue.put(self.wrap_result(task_id, short_status)) if not lib.Options().args.is_force and short_status == 'fail': color_stdout( @@ -354,14 +359,14 @@ def run_loop(self, task_queue, result_queue, fragile_retries): raise VoluntaryStopException() self.task_done(task_queue) - def run_all(self, task_queue, result_queue, fragile_retries): + def run_all(self, task_queue, result_queue, fragile_retries, fragile_checksums): if not self.initialized: self.flush_all_tasks(task_queue, result_queue) result_queue.put(self.done_marker()) return try: - self.run_loop(task_queue, result_queue, fragile_retries) + self.run_loop(task_queue, result_queue, fragile_retries, fragile_checksums) except (KeyboardInterrupt, Exception) as e: if not isinstance(e, KeyboardInterrupt) and \ not isinstance(e, VoluntaryStopException):