diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index b6093fbc6c..8910e64c16 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -113,6 +113,10 @@ def _makedir(self, *dirs, wipeout=False): os.makedirs(ret, exist_ok=True) return ret + def _run_suffix(self): + current_run = runtime().current_run + return '_%s' % current_run if current_run > 0 else '' + @property def timestamp(self): return self._timestamp.strftime(self.timefmt) if self.timefmt else '' @@ -121,17 +125,22 @@ def timestamp(self): def output_prefix(self): """The output prefix directory of ReFrame.""" if self.outputdir is None: - return os.path.join(self.prefix, 'output', self.timestamp) + return os.path.join(self.prefix, 'output' + self._run_suffix(), + self.timestamp) else: - return os.path.join(self.outputdir, self.timestamp) + return os.path.join(self.outputdir + self._run_suffix(), + self.timestamp) + @property def stage_prefix(self): """The stage prefix directory of ReFrame.""" if self.stagedir is None: - return os.path.join(self.prefix, 'stage', self.timestamp) + return os.path.join(self.prefix, 'stage' + self._run_suffix(), + self.timestamp) else: - return os.path.join(self.stagedir, self.timestamp) + return os.path.join(self.stagedir + self._run_suffix(), + self.timestamp) @property def perflog_prefix(self): @@ -177,6 +186,7 @@ def __init__(self, dict_config, sysdescr=None): self._system.outputdir, self._system.perflogdir) self._modules_system = ModulesSystem.create( self._system.modules_system) + self._current_run = 0 def _autodetect_system(self): """Auto-detect system.""" @@ -204,6 +214,13 @@ def mode(self, name): except KeyError: raise ConfigError('unknown execution mode: %s' % name) from None + def next_run(self): + self._current_run += 1 + + @property + def current_run(self): + return self._current_run + @property def system(self): """The current host system. diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py index 87629078b6..20a7f156a5 100644 --- a/reframe/frontend/executors/__init__.py +++ b/reframe/frontend/executors/__init__.py @@ -150,7 +150,6 @@ def __init__(self, policy, printer=None, max_retries=0): self._policy = policy self._printer = printer or PrettyPrinter() self._max_retries = max_retries - self._current_run = 0 self._stats = TestStats() self._policy.stats = self._stats self._policy.printer = self._printer @@ -207,24 +206,19 @@ def _environ_supported(self, check, environ): return ret and check.supports_environ(environ.name) def _retry_failed(self, checks): + rt = runtime.runtime() while (self._stats.num_failures() and - self._current_run < self._max_retries): + rt.current_run < self._max_retries): failed_checks = [ c for c in checks if c.name in set([t.check.name for t in self._stats.tasks_failed()]) ] - self._current_run += 1 - self._stats.next_run() - if self._stats.current_run != self._current_run: - raise AssertionError('current_run variable out of sync' - '(Runner: %d; TestStats: %d)' % - self._current_run, - self._stats.current_run) + rt.next_run() self._printer.separator( 'short double line', 'Retrying %d failed check(s) (retry %d/%d)' % - (len(failed_checks), self._current_run, self._max_retries) + (len(failed_checks), rt.current_run, self._max_retries) ) self._runall(failed_checks) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 5b611e062b..19353d24a5 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -1,5 +1,5 @@ import reframe.core.debug as debug - +import reframe.core.runtime as rt from reframe.core.exceptions import StatisticsError @@ -9,21 +9,16 @@ class TestStats: def __init__(self): # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] self._tasks = [[]] - self._current_run = 0 def __repr__(self): return debug.repr(self) - @property - def current_run(self): - return self._current_run - - def next_run(self): - self._current_run += 1 - self._tasks.append([]) - def add_task(self, task): - self._tasks[self._current_run].append(task) + current_run = rt.runtime().current_run + if current_run == len(self._tasks): + self._tasks.append([]) + + self._tasks[current_run].append(task) def get_tasks(self, run=-1): try: @@ -42,7 +37,7 @@ def tasks_failed(self, run=-1): def retry_report(self): # Return an empty report if no retries were done. - if not self._current_run: + if not rt.runtime().current_run: return '' line_width = 78 @@ -71,14 +66,15 @@ def failure_report(self): line_width = 78 report = [line_width * '='] report.append('SUMMARY OF FAILURES') - for tf in (t for t in self.get_tasks(self._current_run) if t.failed): + current_run = rt.runtime().current_run + for tf in (t for t in self.get_tasks(current_run) if t.failed): check = tf.check partition = check.current_partition partname = partition.fullname if partition else 'None' environ_name = (check.current_environ.name if check.current_environ else 'None') - retry_info = ('(for the last of %s retries)' % self._current_run - if self._current_run > 0 else '') + retry_info = ('(for the last of %s retries)' % current_run + if current_run > 0 else '') report.append(line_width * '-') report.append('FAILURE INFO for %s %s' % (check.name, retry_info)) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 49d41b9ac7..fd3e216c61 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -8,6 +8,7 @@ import reframe.utility.os_ext as os_ext from reframe.core.exceptions import JobNotStartedError from reframe.frontend.loader import RegressionCheckLoader +import unittests.fixtures as fixtures from unittests.resources.checks.hellocheck import HelloTest from unittests.resources.checks.frontend_checks import ( KeyboardInterruptCheck, SleepCheck, @@ -26,6 +27,9 @@ def setUp(self): # Set runtime prefix rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests') + # Reset current_run + rt.runtime()._current_run = 0 + def tearDown(self): os_ext.rmtree(rt.runtime().resources.prefix) @@ -140,7 +144,7 @@ def test_retries_bad_check(self): # Ensure that the test was retried #max_retries times and failed. self.assertEqual(1, self.runner.stats.num_cases()) - self.assertEqual(max_retries, self.runner.stats.current_run) + self.assertEqual(max_retries, rt.runtime().current_run) self.assertEqual(1, self.runner.stats.num_failures()) def test_retries_good_check(self): @@ -151,7 +155,7 @@ def test_retries_good_check(self): # Ensure that the test passed without retries. self.assertEqual(1, self.runner.stats.num_cases()) - self.assertEqual(0, self.runner.stats.current_run) + self.assertEqual(0, rt.runtime().current_run) self.assertEqual(0, self.runner.stats.num_failures()) def test_pass_in_retries(self): @@ -169,7 +173,7 @@ def test_pass_in_retries(self): # Ensure that the test passed after retries in run #run_to_pass. self.assertEqual(1, self.runner.stats.num_cases()) self.assertEqual(1, self.runner.stats.num_failures(run=0)) - self.assertEqual(run_to_pass, self.runner.stats.current_run) + self.assertEqual(run_to_pass, rt.runtime().current_run) self.assertEqual(0, self.runner.stats.num_failures()) os.remove(fp.name)