[feat] Use different stage and output directories for different retries of the same test

reframe/core/runtime.py

@@ -113,6 +113,10 @@ def _makedir(self, *dirs, wipeout=False):
         os.makedirs(ret, exist_ok=True)
         return ret
 
+    def _run_suffix(self):
+        current_run = runtime().current_run
+        return '_%s' % current_run if current_run > 0 else ''
+
     @property
     def timestamp(self):
         return self._timestamp.strftime(self.timefmt) if self.timefmt else ''
@@ -121,17 +125,22 @@ def timestamp(self):
     def output_prefix(self):
         """The output prefix directory of ReFrame."""
         if self.outputdir is None:
-            return os.path.join(self.prefix, 'output', self.timestamp)
+            return os.path.join(self.prefix, 'output' + self._run_suffix(),
+                                self.timestamp)
         else:
-            return os.path.join(self.outputdir, self.timestamp)
+            return os.path.join(self.outputdir + self._run_suffix(),
+                                self.timestamp)
+
 
     @property
     def stage_prefix(self):
         """The stage prefix directory of ReFrame."""
         if self.stagedir is None:
-            return os.path.join(self.prefix, 'stage', self.timestamp)
+            return os.path.join(self.prefix, 'stage' + self._run_suffix(),
+                                self.timestamp)
         else:
-            return os.path.join(self.stagedir, self.timestamp)
+            return os.path.join(self.stagedir + self._run_suffix(),
+                                self.timestamp)
 
     @property
     def perflog_prefix(self):
@@ -177,6 +186,7 @@ def __init__(self, dict_config, sysdescr=None):
             self._system.outputdir, self._system.perflogdir)
         self._modules_system = ModulesSystem.create(
             self._system.modules_system)
+        self._current_run = 0
 
     def _autodetect_system(self):
         """Auto-detect system."""
@@ -204,6 +214,13 @@ def mode(self, name):
         except KeyError:
             raise ConfigError('unknown execution mode: %s' % name) from None
 
+    def next_run(self):
+        self._current_run += 1
+
+    @property
+    def current_run(self):
+        return self._current_run
+
     @property
     def system(self):
         """The current host system.

reframe/frontend/executors/__init__.py

@@ -150,7 +150,6 @@ def __init__(self, policy, printer=None, max_retries=0):
         self._policy = policy
         self._printer = printer or PrettyPrinter()
         self._max_retries = max_retries
-        self._current_run = 0
         self._stats = TestStats()
         self._policy.stats = self._stats
         self._policy.printer = self._printer
@@ -207,24 +206,19 @@ def _environ_supported(self, check, environ):
             return ret and check.supports_environ(environ.name)
 
     def _retry_failed(self, checks):
+        rt = runtime.runtime()
         while (self._stats.num_failures() and
-               self._current_run < self._max_retries):
+               rt.current_run < self._max_retries):
             failed_checks = [
                 c for c in checks if c.name in
                 set([t.check.name for t in self._stats.tasks_failed()])
             ]
-            self._current_run += 1
-            self._stats.next_run()
-            if self._stats.current_run != self._current_run:
-                raise AssertionError('current_run variable out of sync'
-                                     '(Runner: %d; TestStats: %d)' %
-                                     self._current_run,
-                                     self._stats.current_run)
+            rt.next_run()
 
             self._printer.separator(
                 'short double line',
                 'Retrying %d failed check(s) (retry %d/%d)' %
-                (len(failed_checks), self._current_run, self._max_retries)
+                (len(failed_checks), rt.current_run, self._max_retries)
             )
             self._runall(failed_checks)
 

reframe/frontend/statistics.py

@@ -1,5 +1,5 @@
 import reframe.core.debug as debug
-
+import reframe.core.runtime as rt
 from reframe.core.exceptions import StatisticsError
 
 
@@ -9,21 +9,16 @@ class TestStats:
     def __init__(self):
         # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...]
         self._tasks = [[]]
-        self._current_run = 0
 
     def __repr__(self):
         return debug.repr(self)
 
-    @property
-    def current_run(self):
-        return self._current_run
-
-    def next_run(self):
-        self._current_run += 1
-        self._tasks.append([])
-
     def add_task(self, task):
-        self._tasks[self._current_run].append(task)
+        current_run = rt.runtime().current_run
+        if current_run == len(self._tasks):
+            self._tasks.append([])
+
+        self._tasks[current_run].append(task)
 
     def get_tasks(self, run=-1):
         try:
@@ -42,7 +37,7 @@ def tasks_failed(self, run=-1):
 
     def retry_report(self):
         # Return an empty report if no retries were done.
-        if not self._current_run:
+        if not rt.runtime().current_run:
             return ''
 
         line_width = 78
@@ -71,14 +66,15 @@ def failure_report(self):
         line_width = 78
         report = [line_width * '=']
         report.append('SUMMARY OF FAILURES')
-        for tf in (t for t in self.get_tasks(self._current_run) if t.failed):
+        current_run = rt.runtime().current_run
+        for tf in (t for t in self.get_tasks(current_run) if t.failed):
             check = tf.check
             partition = check.current_partition
             partname = partition.fullname if partition else 'None'
             environ_name = (check.current_environ.name
                             if check.current_environ else 'None')
-            retry_info = ('(for the last of %s retries)' % self._current_run
-                          if self._current_run > 0 else '')
+            retry_info = ('(for the last of %s retries)' % current_run
+                          if current_run > 0 else '')
 
             report.append(line_width * '-')
             report.append('FAILURE INFO for %s %s' % (check.name, retry_info))

unittests/test_policies.py

@@ -8,6 +8,7 @@
 import reframe.utility.os_ext as os_ext
 from reframe.core.exceptions import JobNotStartedError
 from reframe.frontend.loader import RegressionCheckLoader
+import unittests.fixtures as fixtures
 from unittests.resources.checks.hellocheck import HelloTest
 from unittests.resources.checks.frontend_checks import (
     KeyboardInterruptCheck, SleepCheck,
@@ -26,6 +27,9 @@ def setUp(self):
         # Set runtime prefix
         rt.runtime().resources.prefix = tempfile.mkdtemp(dir='unittests')
 
+        # Reset current_run
+        rt.runtime()._current_run = 0
+
     def tearDown(self):
         os_ext.rmtree(rt.runtime().resources.prefix)
 
@@ -140,7 +144,7 @@ def test_retries_bad_check(self):
 
         # Ensure that the test was retried #max_retries times and failed.
         self.assertEqual(1, self.runner.stats.num_cases())
-        self.assertEqual(max_retries, self.runner.stats.current_run)
+        self.assertEqual(max_retries, rt.runtime().current_run)
         self.assertEqual(1, self.runner.stats.num_failures())
 
     def test_retries_good_check(self):
@@ -151,7 +155,7 @@ def test_retries_good_check(self):
 
         # Ensure that the test passed without retries.
         self.assertEqual(1, self.runner.stats.num_cases())
-        self.assertEqual(0, self.runner.stats.current_run)
+        self.assertEqual(0, rt.runtime().current_run)
         self.assertEqual(0, self.runner.stats.num_failures())
 
     def test_pass_in_retries(self):
@@ -169,7 +173,7 @@ def test_pass_in_retries(self):
         # Ensure that the test passed after retries in run #run_to_pass.
         self.assertEqual(1, self.runner.stats.num_cases())
         self.assertEqual(1, self.runner.stats.num_failures(run=0))
-        self.assertEqual(run_to_pass, self.runner.stats.current_run)
+        self.assertEqual(run_to_pass, rt.runtime().current_run)
         self.assertEqual(0, self.runner.stats.num_failures())
         os.remove(fp.name)