From 01dde9ff5b79626bfa5175a6f4d8cd8b5f33ecb9 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Thu, 15 Oct 2020 23:57:04 +0200
Subject: [PATCH 01/29] add restart from json report

---
 reframe/core/pipeline.py               | 17 ++++++
 reframe/frontend/cli.py                | 82 ++++++++++++++++++++++----
 reframe/frontend/executors/__init__.py | 26 +++++++-
 reframe/frontend/executors/policies.py |  3 +-
 reframe/frontend/statistics.py         |  7 ++-
 reframe/schemas/runreport.json         |  2 +-
 6 files changed, 120 insertions(+), 17 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index f5dc3654e2..d01bad58f7 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -16,6 +16,7 @@
 import functools
 import inspect
 import itertools
+import json
 import numbers
 import os
 import shutil
@@ -830,6 +831,22 @@ def _rfm_init(self, name=None, prefix=None):
             # Just an empty environment
             self._cdt_environ = env.Environment('__rfm_cdt_environ')
 
+    def __rfm_json_encode__(self):
+        dump_dict = {
+            'modules': self.modules,
+            'variables': self.variables
+        }
+        return dump_dict
+
+    def restore(self, stagedir):
+        self._stagedir = stagedir
+        json_check = os.path.join(self._stagedir, 'rfm_check.json')
+        with open(json_check, 'r') as f:
+            json_check = json.load(f)
+ 
+        self.modules = json_check['modules']
+        self.variables = json_check['variables']
+
     # Export read-only views to interesting fields
     @property
     def current_environ(self):
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 7501d1f864..86cf2b0c3c 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -5,6 +5,7 @@
 
 import inspect
 import json
+import jsonschema
 import os
 import re
 import socket
@@ -291,6 +292,10 @@ def main():
         help='Set the maximum number of times a failed regression test '
              'may be retried (default: 0)'
     )
+    run_options.add_argument(
+        '--retry-failed', metavar='NUM', action='store', default=None,
+        help='Retry failed tests in a given runreport'
+    )
     run_options.add_argument(
         '--flex-alloc-nodes', action='store',
         dest='flex_alloc_nodes', metavar='{all|STATE|NUM}', default=None,
@@ -537,12 +542,53 @@ def main():
     printer.debug(format_env(options.env_vars))
 
     # Setup the check loader
-    loader = RegressionCheckLoader(
-        load_path=site_config.get('general/0/check_search_path'),
-        recurse=site_config.get('general/0/check_search_recursive'),
-        ignore_conflicts=site_config.get('general/0/ignore_check_conflicts')
-    )
+    if options.retry_failed:
+        with open(options.retry_failed) as f:
+            try:
+                restart_report = json.load(f)
+            except json.JSONDecodeError as e:
+                raise ReframeFatalError(
+                    f"invalid runreport: '{restart_report}'"
+                ) from e
+
+        schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe',
+                                       'schemas', 'runreport.json')
+        with open(schema_filename) as f:
+            try:
+                schema = json.load(f)
+            except json.JSONDecodeError as e:
+                raise ReframeFatalError(
+                    f"invalid schema: '{schema_filename}'"
+                ) from e
 
+        try:
+            jsonschema.validate(restart_report, schema)
+        except jsonschema.ValidationError as e:
+            raise ValueError(f"could not validate restart runreport: "
+                             f"'{restart_report}'") from e
+
+        failed_checks = set()
+        failed_checks_prefixes = set()
+        # for run in restart_report['runs']:
+        for testcase in restart_report['runs'][-1]['testcases']:
+            if testcase['result'] == 'failure':
+                failed_checks.add(hash(testcase['name']) ^
+                                  hash(testcase['system']) ^
+                                  hash(testcase['environment']))
+                failed_checks_prefixes.add(testcase['prefix'])
+
+        loader = RegressionCheckLoader(
+            load_path=site_config.get('general/0/check_search_path'),  #failed_checks_prefixes,
+            ignore_conflicts=site_config.get(
+                'general/0/ignore_check_conflicts')
+        )
+    else:
+        loader = RegressionCheckLoader(
+            load_path=site_config.get('general/0/check_search_path'),
+            recurse=site_config.get('general/0/check_search_recursive'),
+            ignore_conflicts=site_config.get(
+                'general/0/ignore_check_conflicts')
+        )
     def print_infoline(param, value):
         param = param + ':'
         printer.info(f"  {param.ljust(18)} {value}")
@@ -633,13 +679,26 @@ def print_infoline(param, value):
             for h in options.hooks:
                 type(c).disable_hook(h)
 
-        testcases = generate_testcases(checks_matched,
+        testcases_og = generate_testcases(checks_matched,
                                        options.skip_system_check,
                                        options.skip_prgenv_check,
                                        allowed_environs)
-        testgraph = dependency.build_deps(testcases)
-        dependency.validate_deps(testgraph)
-        testcases = dependency.toposort(testgraph)
+        if options.retry_failed:
+            failed_cases = [tc for tc in testcases_og
+                            if tc.__hash__() in failed_checks]
+
+            cases_graph = dependency.build_deps(failed_cases, testcases_og)
+            testcases = dependency.toposort(cases_graph, is_subgraph=True)
+            restored_tests = set()
+            for c in testcases:
+                for d in c.deps:
+                    if d.__hash__() not in failed_checks:
+                        restored_tests.add(d)
+
+        else:
+            testgraph = dependency.build_deps(testcases_og)
+            dependency.validate_deps(testgraph)
+            testcases = dependency.toposort(testgraph)
 
         # Manipulate ReFrame's environment
         if site_config.get('general/0/purge_environment'):
@@ -721,7 +780,10 @@ def print_infoline(param, value):
                 session_info['time_start'] = time.strftime(
                     '%FT%T%z', time.localtime(time_start),
                 )
-                runner.runall(testcases)
+                if options.retry_failed:
+                    runner.restore(restored_tests, restart_report)
+
+                runner.runall(testcases, testcases_og)
             finally:
                 time_end = time.time()
                 session_info['time_end'] = time.strftime(
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 4b2b9accfe..fcbfa691c0 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -5,6 +5,7 @@
 
 import abc
 import copy
+import os
 import signal
 import sys
 import time
@@ -14,12 +15,14 @@
 import reframe.core.logging as logging
 import reframe.core.runtime as runtime
 import reframe.frontend.dependency as dependency
+import reframe.utility.json as jsonext
 from reframe.core.exceptions import (AbortTaskError, JobNotStartedError,
                                      ReframeForceExitError, TaskExit)
 from reframe.core.schedulers.local import LocalJobScheduler
 from reframe.frontend.printer import PrettyPrinter
 from reframe.frontend.statistics import TestStats
 
+
 ABORT_REASONS = (KeyboardInterrupt, ReframeForceExitError, AssertionError)
 
 
@@ -287,6 +290,11 @@ def performance(self):
         self._safe_call(self.check.performance)
 
     def finalize(self):
+        json_check = os.path.join(self.check.stagedir, 'rfm_check.json')
+        print(json_check)
+        with open(json_check, 'w') as f:
+            jsonext.dump(self.check, f)
+
         self._current_stage = 'finalize'
         self._notify_listeners('on_task_success')
 
@@ -366,7 +374,7 @@ def policy(self):
     def stats(self):
         return self._stats
 
-    def runall(self, testcases):
+    def runall(self, testcases, testcases_og=None):
         num_checks = len({tc.check.name for tc in testcases})
         self._printer.separator('short double line',
                                 'Running %d check(s)' % num_checks)
@@ -375,7 +383,10 @@ def runall(self, testcases):
         try:
             self._runall(testcases)
             if self._max_retries:
-                self._retry_failed(testcases)
+                if testcases_og:
+                    self._retry_failed(testcases_og)
+                else:
+                    self._retry_failed(testcases)
 
         finally:
             # Print the summary line
@@ -407,6 +418,17 @@ def _retry_failed(self, cases):
             self._runall(failed_cases)
             failures = self._stats.failures()
 
+    def restore(self, testcases, retry_report):
+        index = {}
+        for run in retry_report['runs']:
+            for t in run['testcases']:
+                index[(t['name'], t['system'], t['environment'])] = t['stagedir']
+
+        for t in testcases:
+            idx = (t.check.name, t.partition.fullname, t.environ.name)
+            #RegressionTask(t).check._stagedir = index[idx]
+            RegressionTask(t).check.restore(index[idx])
+
     def _runall(self, testcases):
         def print_separator(check, prefix):
             self._printer.separator(
diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py
index 0e075ea7fd..5450b6ecc4 100644
--- a/reframe/frontend/executors/policies.py
+++ b/reframe/frontend/executors/policies.py
@@ -241,7 +241,8 @@ def deps_failed(self, task):
         return any(self._task_index[c].failed for c in task.testcase.deps)
 
     def deps_succeeded(self, task):
-        return all(self._task_index[c].succeeded for c in task.testcase.deps)
+        return all((c not in self._task_index or self._task_index[c].succeeded)
+                   for c in task.testcase.deps)
 
     def on_task_setup(self, task):
         partname = task.check.current_partition.fullname
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index 1e2a44b91b..febb90eebe 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -82,6 +82,7 @@ def json(self, force=False):
                     'build_stderr': None,
                     'build_stdout': None,
                     'description': check.descr,
+                    'prefix': check.prefix,
                     'environment': None,
                     'fail_reason': None,
                     'fail_phase': None,
@@ -94,7 +95,7 @@ def json(self, force=False):
                     'outputdir': None,
                     'perfvars': None,
                     'result': None,
-                    'stagedir': None,
+                    'stagedir': check.stagedir,
                     'scheduler': None,
                     'system': check.current_system.name,
                     'tags': list(check.tags),
@@ -115,7 +116,7 @@ def json(self, force=False):
                 entry['scheduler'] = partition.scheduler.registered_name
                 entry['environment'] = environ.name
                 if check.job:
-                    entry['jobid'] = check.job.jobid
+                    entry['jobid'] = f'{check.job.jobid}'
                     entry['job_stderr'] = check.stderr.evaluate()
                     entry['job_stdout'] = check.stdout.evaluate()
                     entry['nodelist'] = check.job.nodelist or []
@@ -127,7 +128,7 @@ def json(self, force=False):
                 if t.failed:
                     num_failures += 1
                     entry['result'] = 'failure'
-                    entry['stagedir'] = check.stagedir
+                    # entry['stagedir'] = check.stagedir
                     entry['fail_phase'] = t.failed_stage
                     if t.exc_info is not None:
                         entry['fail_reason'] = format_exception(*t.exc_info)
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 08de4bd591..2eb0650e8f 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -43,7 +43,7 @@
                                 "environment": {"type": ["string", "null"]},
                                 "fail_phase": {"type": ["string", "null"]},
                                 "fail_reason": {"type": ["string", "null"]},
-                                "jobid": {"type": ["number", "null"]},
+                                "jobid": {"type": ["string", "null"]},
                                 "job_stderr": {"type": ["string", "null"]},
                                 "job_stdout": {"type": ["string", "null"]},
                                 "name": {"type": "string"},

From 9b348e711a92920b9230686bd21c802ded835776 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Tue, 20 Oct 2020 09:29:33 +0200
Subject: [PATCH 02/29] add json encoder

---
 reframe/core/pipeline.py               | 19 +++++++++----------
 reframe/frontend/executors/__init__.py | 10 +++++-----
 reframe/utility/json.py                | 22 ++++++++++++++++++++++
 3 files changed, 36 insertions(+), 15 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index d01bad58f7..d459fec9a8 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -833,19 +833,18 @@ def _rfm_init(self, name=None, prefix=None):
 
     def __rfm_json_encode__(self):
         dump_dict = {
-            'modules': self.modules,
-            'variables': self.variables
+            'rfm_properties': {
+                'modules': self.modules,
+                'variables': self.variables,
+                'stagedir': self.stagedir,
+            }
         }
         return dump_dict
 
-    def restore(self, stagedir):
-        self._stagedir = stagedir
-        json_check = os.path.join(self._stagedir, 'rfm_check.json')
-        with open(json_check, 'r') as f:
-            json_check = json.load(f)
- 
-        self.modules = json_check['modules']
-        self.variables = json_check['variables']
+    def __rfm_json_restore__(self, dump_dict):
+        self.modules = dump_dict['modules']
+        self.variables = dump_dict['variables']
+        self._stagedir = dump_dict['stagedir']
 
     # Export read-only views to interesting fields
     @property
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index fcbfa691c0..8bf4c22ff9 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -291,7 +291,6 @@ def performance(self):
 
     def finalize(self):
         json_check = os.path.join(self.check.stagedir, 'rfm_check.json')
-        print(json_check)
         with open(json_check, 'w') as f:
             jsonext.dump(self.check, f)
 
@@ -419,15 +418,16 @@ def _retry_failed(self, cases):
             failures = self._stats.failures()
 
     def restore(self, testcases, retry_report):
-        index = {}
+        stagedirs = {}
         for run in retry_report['runs']:
             for t in run['testcases']:
-                index[(t['name'], t['system'], t['environment'])] = t['stagedir']
+                idx = (t['name'], t['system'], t['environment'])
+                stagedirs[idx] = t['stagedir']
 
         for t in testcases:
             idx = (t.check.name, t.partition.fullname, t.environ.name)
-            #RegressionTask(t).check._stagedir = index[idx]
-            RegressionTask(t).check.restore(index[idx])
+            with open(os.path.join(stagedirs[idx], 'rfm_check.json')) as f:
+                jsonext.load(f, rfm_obj=RegressionTask(t).check)
 
     def _runall(self, testcases):
         def print_separator(check, prefix):
diff --git a/reframe/utility/json.py b/reframe/utility/json.py
index 2d7e4675f7..06d4e82195 100644
--- a/reframe/utility/json.py
+++ b/reframe/utility/json.py
@@ -22,3 +22,25 @@ def dump(obj, fp, **kwargs):
 def dumps(obj, **kwargs):
     kwargs['cls'] = _ReframeJsonEncoder
     return json.dumps(obj, **kwargs)
+
+
+class _ReframeJsonDecoder(json.JSONDecoder):
+    def __init__(self, *args, **kwargs):
+        if 'rfm_obj' in kwargs:
+            self.rfm_obj = kwargs['rfm_obj']
+            del kwargs['rfm_obj']
+
+        json.JSONDecoder.__init__(self, object_hook=self.object_hook,
+                                  *args, **kwargs)
+
+    def object_hook(self, obj):
+        if 'rfm_properties' in obj:
+            self.rfm_obj.__rfm_json_restore__(obj['rfm_properties'])
+            return self.rfm_obj
+
+        return obj
+
+
+def load(fp, **kwargs):
+    kwargs['cls'] = _ReframeJsonDecoder
+    return json.load(fp, **kwargs)

From 2e81020c23e0d8dd86df06d41e7beb4bc65160f2 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Fri, 23 Oct 2020 10:55:21 +0200
Subject: [PATCH 03/29] add unit test

---
 unittests/test_cli.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 4ae49ecdbf..37d488d552 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -152,6 +152,36 @@ def test_check_success(run_reframe, tmp_path, logfile):
     assert os.path.exists(tmp_path / 'report.json')
 
 
+def test_check_retry_failed(run_reframe, tmp_path, logfile):
+    returncode, stdout, _ = run_reframe(
+        checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
+    )
+    assert 'T0' in stdout
+    assert 'T1' in stdout
+    assert 'T2' in stdout
+    assert 'T3' in stdout
+    assert 'T4' in stdout
+    assert 'T5' in stdout
+    assert 'T6' in stdout
+    assert 'T7' in stdout
+    assert 'T8' in stdout
+    assert 'T9' in stdout
+    returncode, stdout, _ = run_reframe(
+        checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
+        more_options=['--retry-failed', f'{tmp_path}/report.json']
+    )
+    assert 'T0' not in stdout
+    assert 'T1' not in stdout
+    assert 'T2' in stdout
+    assert 'T3' not in stdout
+    assert 'T4' not in stdout
+    assert 'T5' not in stdout
+    assert 'T6' not in stdout
+    assert 'T7' in stdout
+    assert 'T8' in stdout
+    assert 'T9' in stdout
+
+
 def test_check_success_force_local(run_reframe, tmp_path, logfile):
     # We explicitly use a system here with a non-local scheduler and pass the
     # `--force-local` option

From 467e686de2fd5213a920735724ebd181e55edf47 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Tue, 3 Nov 2020 09:30:06 +0100
Subject: [PATCH 04/29] fix comments 1

---
 reframe/core/pipeline.py               |  12 ++-
 reframe/frontend/cli.py                | 113 ++++++++++++++-----------
 reframe/frontend/executors/__init__.py |   8 +-
 reframe/frontend/executors/policies.py |   5 +-
 reframe/frontend/statistics.py         |   4 +-
 5 files changed, 77 insertions(+), 65 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index d459fec9a8..4e85cf5ef6 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -16,7 +16,6 @@
 import functools
 import inspect
 import itertools
-import json
 import numbers
 import os
 import shutil
@@ -832,19 +831,18 @@ def _rfm_init(self, name=None, prefix=None):
             self._cdt_environ = env.Environment('__rfm_cdt_environ')
 
     def __rfm_json_encode__(self):
-        dump_dict = {
+        return {
             'rfm_properties': {
                 'modules': self.modules,
                 'variables': self.variables,
                 'stagedir': self.stagedir,
             }
         }
-        return dump_dict
 
-    def __rfm_json_restore__(self, dump_dict):
-        self.modules = dump_dict['modules']
-        self.variables = dump_dict['variables']
-        self._stagedir = dump_dict['stagedir']
+    def __rfm_json_restore__(self, jsonobj):
+        self.modules = jsonobj['modules']
+        self.variables = jsonobj['variables']
+        self._stagedir = jsonobj['stagedir']
 
     # Export read-only views to interesting fields
     @property
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 86cf2b0c3c..872fc698b0 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -82,6 +82,44 @@ def fmt_list(x):
     return '\n'.join(lines)
 
 
+def get_report_file(report_filename):
+    with open(report_filename) as fp:
+        try:
+            restart_report = json.load(fp)
+        except json.JSONDecodeError as e:
+            raise ReframeError(
+                f"could not load report file: '{restart_report!r}'"
+            ) from e
+
+    schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe',
+                                   'schemas', 'runreport.json')
+    with open(schema_filename) as f:
+        try:
+            schema = json.load(f)
+        except json.JSONDecodeError as e:
+            raise ReframeFatalError(
+                f"invalid schema: '{schema_filename}'"
+            ) from e
+
+    try:
+        jsonschema.validate(restart_report, schema)
+    except jsonschema.ValidationError as e:
+        raise ValueError(f"could not validate report file: "
+                         f"'{restart_report!r}'") from e
+
+    return restart_report
+
+def get_failed_checks_from_report(restart_report):
+    failed_checks = set()
+    for testcase in restart_report['runs'][-1]['testcases']:
+        if testcase['result'] == 'failure':
+            failed_checks.add(hash(testcase['name']) ^
+                              hash(testcase['system']) ^
+                              hash(testcase['environment']))
+
+    return failed_checks
+
+
 def format_env(envvars):
     ret = '[ReFrame Environment]\n'
     notset = '<not set>'
@@ -292,9 +330,10 @@ def main():
         help='Set the maximum number of times a failed regression test '
              'may be retried (default: 0)'
     )
-    run_options.add_argument(
-        '--retry-failed', metavar='NUM', action='store', default=None,
-        help='Retry failed tests in a given runreport'
+    output_options.add_argument(
+        '--retry-failed', action='store', nargs='?', const='',
+        metavar='FILE',
+        help='Retry failed tests in a given runreport '
     )
     run_options.add_argument(
         '--flex-alloc-nodes', action='store',
@@ -542,53 +581,27 @@ def main():
     printer.debug(format_env(options.env_vars))
 
     # Setup the check loader
-    if options.retry_failed:
-        with open(options.retry_failed) as f:
-            try:
-                restart_report = json.load(f)
-            except json.JSONDecodeError as e:
-                raise ReframeFatalError(
-                    f"invalid runreport: '{restart_report}'"
-                ) from e
-
-        schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe',
-                                       'schemas', 'runreport.json')
-        with open(schema_filename) as f:
-            try:
-                schema = json.load(f)
-            except json.JSONDecodeError as e:
-                raise ReframeFatalError(
-                    f"invalid schema: '{schema_filename}'"
-                ) from e
+    if options.retry_failed is not None:
+        if options.retry_failed:
+            filename = options.retry_failed
+        else:
+            filename = os_ext.expandvars(
+                site_config.get('general/report_file'))
 
-        try:
-            jsonschema.validate(restart_report, schema)
-        except jsonschema.ValidationError as e:
-            raise ValueError(f"could not validate restart runreport: "
-                             f"'{restart_report}'") from e
-
-        failed_checks = set()
-        failed_checks_prefixes = set()
-        # for run in restart_report['runs']:
-        for testcase in restart_report['runs'][-1]['testcases']:
-            if testcase['result'] == 'failure':
-                failed_checks.add(hash(testcase['name']) ^
-                                  hash(testcase['system']) ^
-                                  hash(testcase['environment']))
-                failed_checks_prefixes.add(testcase['prefix'])
-
-        loader = RegressionCheckLoader(
-            load_path=site_config.get('general/0/check_search_path'),  #failed_checks_prefixes,
-            ignore_conflicts=site_config.get(
-                'general/0/ignore_check_conflicts')
-        )
+        restart_report = get_report_file(filename)
+        failed_checks = get_failed_checks_from_report(restart_report)
+
+        loader_recurse = False
     else:
-        loader = RegressionCheckLoader(
-            load_path=site_config.get('general/0/check_search_path'),
-            recurse=site_config.get('general/0/check_search_recursive'),
-            ignore_conflicts=site_config.get(
-                'general/0/ignore_check_conflicts')
-        )
+        loader_recurse = site_config.get('general/0/check_search_recursive')
+
+    loader = RegressionCheckLoader(
+        load_path=site_config.get('general/0/check_search_path'),
+        recurse=loader_recurse,
+        ignore_conflicts=site_config.get(
+            'general/0/ignore_check_conflicts')
+    )
+
     def print_infoline(param, value):
         param = param + ':'
         printer.info(f"  {param.ljust(18)} {value}")
@@ -683,7 +696,7 @@ def print_infoline(param, value):
                                        options.skip_system_check,
                                        options.skip_prgenv_check,
                                        allowed_environs)
-        if options.retry_failed:
+        if options.retry_failed is not None:
             failed_cases = [tc for tc in testcases_og
                             if tc.__hash__() in failed_checks]
 
@@ -780,7 +793,7 @@ def print_infoline(param, value):
                 session_info['time_start'] = time.strftime(
                     '%FT%T%z', time.localtime(time_start),
                 )
-                if options.retry_failed:
+                if options.retry_failed is not None:
                     runner.restore(restored_tests, restart_report)
 
                 runner.runall(testcases, testcases_og)
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 8bf4c22ff9..b99aff2bfa 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -290,9 +290,9 @@ def performance(self):
         self._safe_call(self.check.performance)
 
     def finalize(self):
-        json_check = os.path.join(self.check.stagedir, 'rfm_check.json')
-        with open(json_check, 'w') as f:
-            jsonext.dump(self.check, f)
+        json_check = os.path.join(self.check.stagedir, '.rfm_testcase.json')
+        with open(json_check, 'w') as fp:
+            jsonext.dump(self.check, fp)
 
         self._current_stage = 'finalize'
         self._notify_listeners('on_task_success')
@@ -426,7 +426,7 @@ def restore(self, testcases, retry_report):
 
         for t in testcases:
             idx = (t.check.name, t.partition.fullname, t.environ.name)
-            with open(os.path.join(stagedirs[idx], 'rfm_check.json')) as f:
+            with open(os.path.join(stagedirs[idx], '.rfm_testcase.json')) as f:
                 jsonext.load(f, rfm_obj=RegressionTask(t).check)
 
     def _runall(self, testcases):
diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py
index 5450b6ecc4..2ba16b7483 100644
--- a/reframe/frontend/executors/policies.py
+++ b/reframe/frontend/executors/policies.py
@@ -241,8 +241,9 @@ def deps_failed(self, task):
         return any(self._task_index[c].failed for c in task.testcase.deps)
 
     def deps_succeeded(self, task):
-        return all((c not in self._task_index or self._task_index[c].succeeded)
-                   for c in task.testcase.deps)
+        return all(self._task_index[c].succeeded
+                   for c in task.testcase.deps
+                   if c in self._task_index)
 
     def on_task_setup(self, task):
         partname = task.check.current_partition.fullname
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index febb90eebe..fc9eac20f0 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -77,6 +77,7 @@ def json(self, force=False):
             num_failures = 0
             for t in run:
                 check = t.check
+                print('\nxxx', check.prefix)
                 partition = check.current_partition
                 entry = {
                     'build_stderr': None,
@@ -116,7 +117,7 @@ def json(self, force=False):
                 entry['scheduler'] = partition.scheduler.registered_name
                 entry['environment'] = environ.name
                 if check.job:
-                    entry['jobid'] = f'{check.job.jobid}'
+                    entry['jobid'] = str(check.job.jobid)
                     entry['job_stderr'] = check.stderr.evaluate()
                     entry['job_stdout'] = check.stdout.evaluate()
                     entry['nodelist'] = check.job.nodelist or []
@@ -128,7 +129,6 @@ def json(self, force=False):
                 if t.failed:
                     num_failures += 1
                     entry['result'] = 'failure'
-                    # entry['stagedir'] = check.stagedir
                     entry['fail_phase'] = t.failed_stage
                     if t.exc_info is not None:
                         entry['fail_reason'] = format_exception(*t.exc_info)

From 901ab1b0f9b5f874863ec50b3ec313fd1f42b607 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Wed, 4 Nov 2020 15:56:07 +0100
Subject: [PATCH 05/29] fix comments 2

---
 reframe/core/pipeline.py       | 8 +++-----
 reframe/frontend/statistics.py | 1 -
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 4e85cf5ef6..0967f785b7 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -832,11 +832,9 @@ def _rfm_init(self, name=None, prefix=None):
 
     def __rfm_json_encode__(self):
         return {
-            'rfm_properties': {
-                'modules': self.modules,
-                'variables': self.variables,
-                'stagedir': self.stagedir,
-            }
+            'modules': self.modules,
+            'variables': self.variables,
+            'stagedir': self.stagedir,
         }
 
     def __rfm_json_restore__(self, jsonobj):
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index fc9eac20f0..8302aa5bb3 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -77,7 +77,6 @@ def json(self, force=False):
             num_failures = 0
             for t in run:
                 check = t.check
-                print('\nxxx', check.prefix)
                 partition = check.current_partition
                 entry = {
                     'build_stderr': None,

From 847d0ae37ebf7cbae855234bf461e4d6b6ace70f Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Wed, 4 Nov 2020 16:41:37 +0100
Subject: [PATCH 06/29] add json fix

---
 reframe/utility/json.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/reframe/utility/json.py b/reframe/utility/json.py
index 06d4e82195..4b10b8aa5a 100644
--- a/reframe/utility/json.py
+++ b/reframe/utility/json.py
@@ -34,8 +34,8 @@ def __init__(self, *args, **kwargs):
                                   *args, **kwargs)
 
     def object_hook(self, obj):
-        if 'rfm_properties' in obj:
-            self.rfm_obj.__rfm_json_restore__(obj['rfm_properties'])
+        if 'modules' in obj:
+            self.rfm_obj.__rfm_json_restore__(obj)
             return self.rfm_obj
 
         return obj

From 5527a81a5192fc905c31f009e396b297a57d1f5a Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Thu, 5 Nov 2020 14:25:20 +0100
Subject: [PATCH 07/29] move restore to cli

---
 reframe/frontend/cli.py                | 41 +++++++++++++++++++-------
 reframe/frontend/executors/__init__.py | 24 ++++++---------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 872fc698b0..6ddcc26b9d 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -28,7 +28,8 @@
     ReframeDeprecationWarning, ReframeFatalError,
     format_exception, SystemAutodetectionError
 )
-from reframe.frontend.executors import Runner, generate_testcases
+from reframe.frontend.executors import (RegressionTask, Runner,
+                                        generate_testcases)
 from reframe.frontend.executors.policies import (SerialExecutionPolicy,
                                                  AsynchronousExecutionPolicy)
 from reframe.frontend.loader import RegressionCheckLoader
@@ -109,6 +110,7 @@ def get_report_file(report_filename):
 
     return restart_report
 
+
 def get_failed_checks_from_report(restart_report):
     failed_checks = set()
     for testcase in restart_report['runs'][-1]['testcases']:
@@ -120,6 +122,24 @@ def get_failed_checks_from_report(restart_report):
     return failed_checks
 
 
+def restore(testcases, retry_report, printer):
+    stagedirs = {}
+    for run in retry_report['runs']:
+        for t in run['testcases']:
+            idx = (t['name'], t['system'], t['environment'])
+            stagedirs[idx] = t['stagedir']
+
+    for i, t in enumerate(testcases):
+        idx = (t.check.name, t.partition.fullname, t.environ.name)
+        try:
+            with open(os.path.join(stagedirs[idx],
+                                   '.rfm_testcase.json')) as f:
+                jsonext.load(f, rfm_obj=RegressionTask(t).check)
+        except (OSError, json.JSONDecodeError):
+            printer.warning(f'check {RegressionTask(t).check.name} '
+                            f'can not be restored')
+
+
 def format_env(envvars):
     ret = '[ReFrame Environment]\n'
     notset = '<not set>'
@@ -692,15 +712,16 @@ def print_infoline(param, value):
             for h in options.hooks:
                 type(c).disable_hook(h)
 
-        testcases_og = generate_testcases(checks_matched,
-                                       options.skip_system_check,
-                                       options.skip_prgenv_check,
-                                       allowed_environs)
+        testcases_unfiltered = generate_testcases(checks_matched,
+                                                  options.skip_system_check,
+                                                  options.skip_prgenv_check,
+                                                  allowed_environs)
         if options.retry_failed is not None:
-            failed_cases = [tc for tc in testcases_og
+            failed_cases = [tc for tc in testcases_unfiltered
                             if tc.__hash__() in failed_checks]
 
-            cases_graph = dependency.build_deps(failed_cases, testcases_og)
+            cases_graph = dependency.build_deps(failed_cases,
+                                                testcases_unfiltered)
             testcases = dependency.toposort(cases_graph, is_subgraph=True)
             restored_tests = set()
             for c in testcases:
@@ -709,7 +730,7 @@ def print_infoline(param, value):
                         restored_tests.add(d)
 
         else:
-            testgraph = dependency.build_deps(testcases_og)
+            testgraph = dependency.build_deps(testcases_unfiltered)
             dependency.validate_deps(testgraph)
             testcases = dependency.toposort(testgraph)
 
@@ -794,9 +815,9 @@ def print_infoline(param, value):
                     '%FT%T%z', time.localtime(time_start),
                 )
                 if options.retry_failed is not None:
-                    runner.restore(restored_tests, restart_report)
+                    restore(restored_tests, restart_report, printer)
 
-                runner.runall(testcases, testcases_og)
+                runner.runall(testcases, testcases_unfiltered)
             finally:
                 time_end = time.time()
                 session_info['time_end'] = time.strftime(
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index b99aff2bfa..47b4dc4baf 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -5,6 +5,7 @@
 
 import abc
 import copy
+import json
 import os
 import signal
 import sys
@@ -290,9 +291,14 @@ def performance(self):
         self._safe_call(self.check.performance)
 
     def finalize(self):
-        json_check = os.path.join(self.check.stagedir, '.rfm_testcase.json')
-        with open(json_check, 'w') as fp:
-            jsonext.dump(self.check, fp)
+        try:
+            json_check = os.path.join(self.check.stagedir,
+                                      '.rfm_testcase.json')
+            with open(json_check, 'w') as fp:
+                jsonext.dump(self.check, fp)
+        except OSError:
+            self._printer.warning(f'check {RegressionTask(t).check.name} '
+                                  f'can not be dumped')
 
         self._current_stage = 'finalize'
         self._notify_listeners('on_task_success')
@@ -417,18 +423,6 @@ def _retry_failed(self, cases):
             self._runall(failed_cases)
             failures = self._stats.failures()
 
-    def restore(self, testcases, retry_report):
-        stagedirs = {}
-        for run in retry_report['runs']:
-            for t in run['testcases']:
-                idx = (t['name'], t['system'], t['environment'])
-                stagedirs[idx] = t['stagedir']
-
-        for t in testcases:
-            idx = (t.check.name, t.partition.fullname, t.environ.name)
-            with open(os.path.join(stagedirs[idx], '.rfm_testcase.json')) as f:
-                jsonext.load(f, rfm_obj=RegressionTask(t).check)
-
     def _runall(self, testcases):
         def print_separator(check, prefix):
             self._printer.separator(

From 1a89b56d9c1177e7bf69d9e351e9127f04aa7e20 Mon Sep 17 00:00:00 2001
From: rafael <rafael.sarmiento@cscs.ch>
Date: Fri, 6 Nov 2020 15:41:16 +0100
Subject: [PATCH 08/29] add unit test

---
 reframe/frontend/cli.py        | 10 ++++++++--
 reframe/frontend/statistics.py |  2 ++
 reframe/schemas/runreport.json |  4 +++-
 unittests/test_cli.py          | 36 +++++++++++++++-------------------
 4 files changed, 29 insertions(+), 23 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 6ddcc26b9d..950d4e5b28 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -122,6 +122,11 @@ def get_failed_checks_from_report(restart_report):
     return failed_checks
 
 
+def get_filenames_from_report(restart_report):
+    return list({testcase['filename']
+                for testcase in restart_report['runs'][-1]['testcases']})
+
+
 def restore(testcases, retry_report, printer):
     stagedirs = {}
     for run in retry_report['runs']:
@@ -610,13 +615,14 @@ def main():
 
         restart_report = get_report_file(filename)
         failed_checks = get_failed_checks_from_report(restart_report)
-
+        load_paths = get_filenames_from_report(restart_report)
         loader_recurse = False
     else:
         loader_recurse = site_config.get('general/0/check_search_recursive')
+        load_paths = site_config.get('general/0/check_search_path')
 
     loader = RegressionCheckLoader(
-        load_path=site_config.get('general/0/check_search_path'),
+        load_path=load_paths,
         recurse=loader_recurse,
         ignore_conflicts=site_config.get(
             'general/0/ignore_check_conflicts')
diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py
index 8302aa5bb3..81998ece02 100644
--- a/reframe/frontend/statistics.py
+++ b/reframe/frontend/statistics.py
@@ -3,6 +3,7 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import inspect
 import reframe.core.runtime as rt
 from reframe.core.exceptions import format_exception, StatisticsError
 
@@ -83,6 +84,7 @@ def json(self, force=False):
                     'build_stdout': None,
                     'description': check.descr,
                     'prefix': check.prefix,
+                    'filename': inspect.getfile(type(check)),
                     'environment': None,
                     'fail_reason': None,
                     'fail_phase': None,
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 2eb0650e8f..3e52531988 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -40,6 +40,8 @@
                                 "build_stderr": {"type": ["string", "null"]},
                                 "build_stdout": {"type": ["string", "null"]},
                                 "description": {"type": "string"},
+                                "prefix": {"type": "string"},
+                                "filename": {"type": "string"},
                                 "environment": {"type": ["string", "null"]},
                                 "fail_phase": {"type": ["string", "null"]},
                                 "fail_reason": {"type": ["string", "null"]},
@@ -100,7 +102,7 @@
                                 "time_total": {"type": ["number", "null"]}
                             },
                             "required": [
-                                "environment", "name", "result", "system"
+                                "environment", "name", "result", "system", "filename"
                             ]
                         },
                         "required": [
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 37d488d552..23a0429b9a 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -4,6 +4,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import itertools
+import json
 import os
 import pathlib
 import pytest
@@ -156,30 +157,25 @@ def test_check_retry_failed(run_reframe, tmp_path, logfile):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
     )
-    assert 'T0' in stdout
-    assert 'T1' in stdout
-    assert 'T2' in stdout
-    assert 'T3' in stdout
-    assert 'T4' in stdout
-    assert 'T5' in stdout
-    assert 'T6' in stdout
-    assert 'T7' in stdout
-    assert 'T8' in stdout
-    assert 'T9' in stdout
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
         more_options=['--retry-failed', f'{tmp_path}/report.json']
     )
-    assert 'T0' not in stdout
-    assert 'T1' not in stdout
-    assert 'T2' in stdout
-    assert 'T3' not in stdout
-    assert 'T4' not in stdout
-    assert 'T5' not in stdout
-    assert 'T6' not in stdout
-    assert 'T7' in stdout
-    assert 'T8' in stdout
-    assert 'T9' in stdout
+    with open(f'{tmp_path}/report.json') as fp:
+        report = json.load(fp)
+
+    report_summary = {t['name']: t['fail_phase']
+        for run in report['runs']
+        for t in run['testcases']
+    }
+
+    assert report_summary['T2'] == 'sanity'
+    assert report_summary['T7'] == 'startup'
+    assert report_summary['T8'] == 'setup'
+    assert report_summary['T9'] == 'startup'
+
+    assert all(i not in report_summary.keys()
+               for i in ['T0', 'T1', 'T3', 'T4', 'T5', 'T6'])
 
 
 def test_check_success_force_local(run_reframe, tmp_path, logfile):

From c9b15c08b2726e6005eafb0fe15f79ddcd0004f4 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 9 Nov 2020 22:03:06 +0100
Subject: [PATCH 09/29] Remove conflict leftovers

---
 reframe/frontend/cli.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 2937744cb3..37c84ef691 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -42,9 +42,6 @@
 import reframe.core.warnings as warnings
 import reframe.frontend.argparse as argparse
 import reframe.frontend.check_filters as filters
-<< << << < HEAD
-== == == =
->>>>>> > master
 
 
 def format_check(check, check_deps, detailed=False):

From 07cd150cbd84769e232f47b1e51693fb20b53f53 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 9 Nov 2020 22:32:03 +0100
Subject: [PATCH 10/29] Fix problems after merging with master

---
 reframe/frontend/cli.py                | 42 +++++++++++---------------
 reframe/frontend/executors/__init__.py |  2 +-
 reframe/schemas/runreport.json         |  2 +-
 unittests/test_cli.py                  | 12 ++++----
 4 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 37c84ef691..ec91e3b4eb 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -3,24 +3,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-from reframe.frontend.printer import PrettyPrinter
-from reframe.frontend.loader import RegressionCheckLoader
-from reframe.frontend.executors.policies import (SerialExecutionPolicy,
-                                                 AsynchronousExecutionPolicy)
-from reframe.frontend.executors import Runner, generate_testcases
-import reframe.utility.osext as osext
-import reframe.utility.jsonext as jsonext
-import reframe.frontend.dependencies as dependencies
-from reframe.frontend.executors import (RegressionTask, Runner,
-                                        generate_testcases)
-from reframe.core.exceptions import (
-    EnvironError, ConfigError, ReframeError,
-    ReframeDeprecationWarning, ReframeFatalError,
-    format_exception, SystemAutodetectionError
-)
-import reframe.utility.json as jsonext
-import reframe.utility.os_ext as os_ext
-import reframe.frontend.dependency as dependency
 import inspect
 import itertools
 import json
@@ -42,6 +24,18 @@
 import reframe.core.warnings as warnings
 import reframe.frontend.argparse as argparse
 import reframe.frontend.check_filters as filters
+import reframe.frontend.dependencies as dependencies
+import reframe.utility.jsonext as jsonext
+import reframe.utility.osext as osext
+
+
+from reframe.frontend.printer import PrettyPrinter
+from reframe.frontend.loader import RegressionCheckLoader
+from reframe.frontend.executors.policies import (SerialExecutionPolicy,
+                                                 AsynchronousExecutionPolicy)
+from reframe.frontend.executors import Runner, generate_testcases
+from reframe.frontend.executors import (RegressionTask, Runner,
+                                        generate_testcases)
 
 
 def format_check(check, check_deps, detailed=False):
@@ -799,9 +793,9 @@ def print_infoline(param, value):
             failed_cases = [tc for tc in testcases_unfiltered
                             if tc.__hash__() in failed_checks]
 
-            cases_graph = dependency.build_deps(failed_cases,
-                                                testcases_unfiltered)
-            testcases = dependency.toposort(cases_graph, is_subgraph=True)
+            cases_graph = dependencies.build_deps(failed_cases,
+                                                  testcases_unfiltered)
+            testcases = dependencies.toposort(cases_graph, is_subgraph=True)
             restored_tests = set()
             for c in testcases:
                 for d in c.deps:
@@ -809,9 +803,9 @@ def print_infoline(param, value):
                         restored_tests.add(d)
 
         else:
-            testgraph = dependency.build_deps(testcases_unfiltered)
-            dependency.validate_deps(testgraph)
-            testcases = dependency.toposort(testgraph)
+            testgraph = dependencies.build_deps(testcases_unfiltered)
+            dependencies.validate_deps(testgraph)
+            testcases = dependencies.toposort(testgraph)
 
         # FIXME: These two should be reconsidered
         printer.debug(f'Generated {len(testcases)} test case(s)')
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index b55aabf2e4..ef3e65e396 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -16,7 +16,7 @@
 import reframe.core.logging as logging
 import reframe.core.runtime as runtime
 import reframe.frontend.dependencies as dependencies
-import reframe.utility.json as jsonext
+import reframe.utility.jsonext as jsonext
 from reframe.core.exceptions import (AbortTaskError, JobNotStartedError,
                                      ReframeForceExitError, TaskExit)
 from reframe.core.schedulers.local import LocalJobScheduler
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index d51abbcbd1..f86f7209e2 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -49,7 +49,7 @@
                                         "exc_type": {"type": "string"},
                                         "exc_value": {"type": "string"},
                                         "traceback": {
-                                            "type": "array",
+                                            "type": ["array", "null"],
                                             "items": {"type": "string"}
                                         }
                                     },
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 5d9f9c1da3..d752d36650 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -146,7 +146,7 @@ def test_check_success(run_reframe, tmp_path):
     assert os.path.exists(tmp_path / 'report.json')
 
 
-def test_check_retry_failed(run_reframe, tmp_path, logfile):
+def test_check_retry_failed(run_reframe, tmp_path):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
     )
@@ -157,11 +157,11 @@ def test_check_retry_failed(run_reframe, tmp_path, logfile):
     with open(f'{tmp_path}/report.json') as fp:
         report = json.load(fp)
 
-    report_summary = {t['name']: t['fail_phase']
-                      for run in report['runs']
-                      for t in run['testcases']
-                      }
-
+    report_summary = {
+        t['name']: t['fail_phase']
+        for run in report['runs']
+        for t in run['testcases']
+    }
     assert report_summary['T2'] == 'sanity'
     assert report_summary['T7'] == 'startup'
     assert report_summary['T8'] == 'setup'

From 551afa3b829f97f10befdc2d201a6e272ea7cacb Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 9 Nov 2020 23:46:49 +0100
Subject: [PATCH 11/29] WIP: Refactoring the `--rerun-failed` implementation

---
 reframe/core/pipeline.py      |   9 ++
 reframe/frontend/cli.py       | 135 +++++++-----------------------
 reframe/frontend/runreport.py | 149 ++++++++++++++++++++++++++++++++++
 3 files changed, 187 insertions(+), 106 deletions(-)
 create mode 100644 reframe/frontend/runreport.py

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index b3c28a2782..6f19954bd7 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -1754,6 +1754,15 @@ def __str__(self):
         return "%s(name='%s', prefix='%s')" % (type(self).__name__,
                                                self.name, self.prefix)
 
+    def __hash__(self):
+        return hash(self.name)
+
+    def __eq__(self, other):
+        if not isinstance(other, RegressionTest):
+            return NotImplemented
+
+        return self.name == other.name
+
 
 class RunOnlyRegressionTest(RegressionTest, special=True):
     '''Base class for run-only regression tests.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index ec91e3b4eb..14365372e6 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -25,6 +25,7 @@
 import reframe.frontend.argparse as argparse
 import reframe.frontend.check_filters as filters
 import reframe.frontend.dependencies as dependencies
+import reframe.frontend.runreport as runreport
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 
@@ -100,68 +101,6 @@ def fmt_deps():
     return '\n'.join(lines)
 
 
-def get_report_file(report_filename):
-    with open(report_filename) as fp:
-        try:
-            restart_report = json.load(fp)
-        except json.JSONDecodeError as e:
-            raise ReframeError(
-                f"could not load report file: '{restart_report!r}'"
-            ) from e
-
-    schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe',
-                                   'schemas', 'runreport.json')
-    with open(schema_filename) as f:
-        try:
-            schema = json.load(f)
-        except json.JSONDecodeError as e:
-            raise ReframeFatalError(
-                f"invalid schema: '{schema_filename}'"
-            ) from e
-
-    try:
-        jsonschema.validate(restart_report, schema)
-    except jsonschema.ValidationError as e:
-        raise ValueError(f"could not validate report file: "
-                         f"'{restart_report!r}'") from e
-
-    return restart_report
-
-
-def get_failed_checks_from_report(restart_report):
-    failed_checks = set()
-    for testcase in restart_report['runs'][-1]['testcases']:
-        if testcase['result'] == 'failure':
-            failed_checks.add(hash(testcase['name']) ^
-                              hash(testcase['system']) ^
-                              hash(testcase['environment']))
-
-    return failed_checks
-
-
-def get_filenames_from_report(restart_report):
-    return list({testcase['filename']
-                 for testcase in restart_report['runs'][-1]['testcases']})
-
-
-def restore(testcases, retry_report, printer):
-    stagedirs = {}
-    for run in retry_report['runs']:
-        for t in run['testcases']:
-            idx = (t['name'], t['system'], t['environment'])
-            stagedirs[idx] = t['stagedir']
-
-    for i, t in enumerate(testcases):
-        idx = (t.check.name, t.partition.fullname, t.environ.name)
-        try:
-            with open(os.path.join(stagedirs[idx],
-                                   '.rfm_testcase.json')) as f:
-                jsonext.load(f, rfm_obj=RegressionTask(t).check)
-        except (OSError, json.JSONDecodeError):
-            printer.warning(f'check {RegressionTask(t).check.name} '
-                            f'can not be restored')
-
-
 def format_env(envvars):
     ret = '[ReFrame Environment]\n'
     notset = '<not set>'
@@ -186,23 +125,6 @@ def list_checks(testcases, printer, detailed=False):
     printer.info(f'Found {len(checks)} check(s)')
 
 
-def generate_report_filename(filepatt):
-    if '{sessionid}' not in filepatt:
-        return filepatt
-
-    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
-    new_id = -1
-    basedir = os.path.dirname(filepatt) or '.'
-    for filename in os.listdir(basedir):
-        match = re.match(search_patt, filename)
-        if match:
-            found_id = int(match.group(1))
-            new_id = max(found_id, new_id)
-
-    new_id += 1
-    return filepatt.format(sessionid=new_id)
-
-
 def logfiles_message():
     log_files = logging.log_files()
     msg = 'Log file(s) saved in: '
@@ -654,23 +576,27 @@ def main():
 
     # Setup the check loader
     if options.retry_failed is not None:
+        # We need to load the failed checks only from a report
         if options.retry_failed:
             filename = options.retry_failed
         else:
-            filename = os_ext.expandvars(
-                site_config.get('general/report_file'))
+            filename = runreport.next_report_filename(
+                osext.expandvars(site_config.get('general/0/report_file')),
+                new=False
+            )
 
-        restart_report = get_report_file(filename)
-        failed_checks = get_failed_checks_from_report(restart_report)
-        load_paths = get_filenames_from_report(restart_report)
-        loader_recurse = False
+        report = runreport.load_report(filename)
+        check_search_path = list(report.slice('filename', unique=True))
+        check_search_recursive = False
     else:
-        loader_recurse = site_config.get('general/0/check_search_recursive')
-        load_paths = site_config.get('general/0/check_search_path')
+        check_search_recursive = site_config.get(
+            'general/0/check_search_recursive'
+        )
+        check_search_path = site_config.get('general/0/check_search_path')
 
     loader = RegressionCheckLoader(
-        load_path=load_paths,
-        recurse=loader_recurse,
+        load_path=check_search_path,
+        recurse=check_search_recursive,
         ignore_conflicts=site_config.get(
             'general/0/ignore_check_conflicts')
     )
@@ -682,7 +608,7 @@ def print_infoline(param, value):
     session_info = {
         'cmdline': ' '.join(sys.argv),
         'config_file': rt.site_config.filename,
-        'data_version': '1.0',
+        'data_version': '1.1',
         'hostname': socket.gethostname(),
         'prefix_output': rt.output_prefix,
         'prefix_stage': rt.stage_prefix,
@@ -790,18 +716,18 @@ def print_infoline(param, value):
                                                   options.skip_prgenv_check,
                                                   allowed_environs)
         if options.retry_failed is not None:
-            failed_cases = [tc for tc in testcases_unfiltered
-                            if tc.__hash__() in failed_checks]
-
-            cases_graph = dependencies.build_deps(failed_cases,
-                                                  testcases_unfiltered)
-            testcases = dependencies.toposort(cases_graph, is_subgraph=True)
-            restored_tests = set()
-            for c in testcases:
-                for d in c.deps:
-                    if d.__hash__() not in failed_checks:
-                        restored_tests.add(d)
-
+            # Filter the cases based on the report
+            failed_cases = []
+            for tc in testcases_unfiltered:
+                case = report.case(*tc)
+                if case and case['result'] == 'failure':
+                    failed_cases.append(tc)
+
+            testgraph = dependencies.build_deps(failed_cases,
+                                                testcases_unfiltered)
+            testcases = dependencies.toposort(testgraph, is_subgraph=True)
+            print(testcases)
+            testcases = report.restore_deps(testcases)
         else:
             testgraph = dependencies.build_deps(testcases_unfiltered)
             dependencies.validate_deps(testgraph)
@@ -898,9 +824,6 @@ def print_infoline(param, value):
                 session_info['time_start'] = time.strftime(
                     '%FT%T%z', time.localtime(time_start),
                 )
-                if options.retry_failed is not None:
-                    restore(restored_tests, restart_report, printer)
-
                 runner.runall(testcases, testcases_unfiltered)
             finally:
                 time_end = time.time()
@@ -941,7 +864,7 @@ def print_infoline(param, value):
                     'session_info': session_info,
                     'runs': run_stats
                 }
-                report_file = generate_report_filename(report_file)
+                report_file = runreport.next_report_filename(report_file)
                 try:
                     with open(report_file, 'w') as fp:
                         jsonext.dump(json_report, fp, indent=2)
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
new file mode 100644
index 0000000000..954b7da7b5
--- /dev/null
+++ b/reframe/frontend/runreport.py
@@ -0,0 +1,149 @@
+# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich)
+# ReFrame Project Developers. See the top-level LICENSE file for details.
+#
+# SPDX-License-Identifier: BSD-3-Clause
+
+import json
+import jsonschema
+import os
+import re
+
+import reframe as rfm
+import reframe.core.exceptions as errors
+import reframe.utility.jsonext as jsonext
+
+
+_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
+_DATA_VERSION = '1.1'
+
+
+class _RunReport:
+    '''A wrapper to the run report providing some additional functionality'''
+
+    def __init__(self, report):
+        self._report = report
+
+        # Index all runs by test case; if a test case has run multiple times,
+        # only the last time will be indexed
+        self._cases_index = {}
+        for run in self._report['runs']:
+            for tc in run['testcases']:
+                c, p, e = tc['name'], tc['system'], tc['environment']
+                self._cases_index[c, p, e] = tc
+
+    def __getattr__(self, name):
+        return getattr(self._report, name)
+
+    def slice(self, prop, when=None, unique=False):
+        '''Slice the report on property ``prop``.'''
+
+        if unique:
+            returned = set()
+
+        for tc in self._report['runs'][-1]['testcases']:
+            val = tc[prop]
+            if unique and val in returned:
+                continue
+
+            if when is None:
+                returned.add(val)
+                yield val
+            elif tc[when[0]] == when[1]:
+                returned.add(val)
+                yield val
+
+    def case(self, check, part, env):
+        c, p, e = check.name, part.fullname, env.name
+        return self._cases_index.get((c, p, e))
+
+    def restore_deps(self, testcases):
+        '''Restore state of successful dependencies in testcases
+
+        Returns the updated tescases.
+        '''
+
+        for tc in testcases:
+            for d in tc.deps:
+                print(d)
+                if self.case(*d)['result'] == 'success':
+                    self._do_restore(d)
+
+        return testcases
+
+    def _do_restore(self, testcase):
+        dump_file = os.path.join(self.case(*testcase)['stagedir'],
+                                 '.rfm_testcase.json')
+        try:
+            with open(dump_file) as fp:
+                jsonext.load(fp, rfm_obj=testcase.check)
+        except (OSError, json.JSONDecodeError) as e:
+            raise errors.ReframeError(
+                f'could not restore testase {testcase!r}') from e
+
+
+def next_report_filename(filepatt, new=True):
+    if '{sessionid}' not in filepatt:
+        return filepatt
+
+    search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)')
+    new_id = -1
+    basedir = os.path.dirname(filepatt) or '.'
+    for filename in os.listdir(basedir):
+        match = re.match(search_patt, filename)
+        if match:
+            found_id = int(match.group(1))
+            new_id = max(found_id, new_id)
+
+    if new:
+        new_id += 1
+
+    return filepatt.format(sessionid=new_id)
+
+
+def load_report(filename):
+    try:
+        with open(filename) as fp:
+            report = json.load(fp)
+    except OSError as e:
+        raise errors.ReframeError(
+            f'failed to load report file {filename!r}') from e
+    except json.JSONDecodeError as e:
+        raise errors.ReframeError(
+            f'report file {filename!r} is not a valid JSON file') from e
+
+    # Validate the report
+    with open(_SCHEMA) as fp:
+        schema = json.load(fp)
+
+    try:
+        jsonschema.validate(report, schema)
+    except jsonschema.ValidationError as e:
+        raise errors.ReframeError(f'invalid report {filename!r}') from e
+
+    # Check if the report data is compatible
+    data_ver = report['session_info']['data_version']
+    if data_ver != _DATA_VERSION:
+        raise errors.ReframeError(
+            f'incompatible report data versions: '
+            f'found {data_ver!r}, required {_DATA_VERSION!r}'
+        )
+
+    return _RunReport(report)
+
+
+def restore(testcases, retry_report, printer):
+    stagedirs = {}
+    for run in retry_report['runs']:
+        for t in run['testcases']:
+            idx = (t['name'], t['system'], t['environment'])
+            stagedirs[idx] = t['stagedir']
+
+    for i, t in enumerate(testcases):
+        idx = (t.check.name, t.partition.fullname, t.environ.name)
+        try:
+            with open(os.path.join(stagedirs[idx],
+                                   '.rfm_testcase.json')) as f:
+                jsonext.load(f, rfm_obj=RegressionTask(t).check)
+        except (OSError, json.JSONDecodeError):
+            printer.warning(f'check {RegressionTask(t).check.name} '
+                            f'can not be restored')

From 0814306d9525518a98da6d0e12bb2649bcd3f06a Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 16 Nov 2020 22:40:05 +0100
Subject: [PATCH 12/29] WIP: Generalize rerun option

---
 reframe/frontend/cli.py          | 58 ++++++++++++++++++++++++++------
 reframe/frontend/dependencies.py | 10 ++++--
 reframe/frontend/runreport.py    | 15 ++++-----
 3 files changed, 61 insertions(+), 22 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 2e4e536fa2..e98edcfd90 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -249,6 +249,10 @@ def main():
         help=('Select checks with at least one '
               'programming environment matching PATTERN')
     )
+    select_options.add_argument(
+        '--failed', action='store_true',
+        help="Select failed test cases (only when '--restore-session' is used)"
+    )
     select_options.add_argument(
         '--gpu-only', action='store_true',
         help='Select only GPU checks'
@@ -315,10 +319,10 @@ def main():
         help='Set the maximum number of times a failed regression test '
              'may be retried (default: 0)'
     )
-    output_options.add_argument(
-        '--retry-failed', action='store', nargs='?', const='',
-        metavar='FILE',
-        help='Retry failed tests in a given runreport '
+    run_options.add_argument(
+        '--restore-session', action='store', nargs='?', const='',
+        metavar='REPORT',
+        help='Restore a testing session from REPORT file'
     )
     run_options.add_argument(
         '--flex-alloc-nodes', action='store',
@@ -575,10 +579,10 @@ def main():
     printer.debug(format_env(options.env_vars))
 
     # Setup the check loader
-    if options.retry_failed is not None:
+    if options.restore_session is not None:
         # We need to load the failed checks only from a report
-        if options.retry_failed:
-            filename = options.retry_failed
+        if options.restore_session:
+            filename = options.restore_session
         else:
             filename = runreport.next_report_filename(
                 osext.expandvars(site_config.get('general/0/report_file')),
@@ -692,6 +696,30 @@ def print_infoline(param, value):
         elif options.cpu_only:
             testcases = filter(filters.have_cpu_only(), testcases)
 
+        testcases = list(testcases)
+        printer.verbose(
+            f'Filtering test cases(s) by other attributes: '
+            f'{len(testcases)} remaining'
+        )
+
+        # Filter in failed cases
+        if options.failed:
+            if options.restore_session is None:
+                printer.error(
+                    "the option '--failed' can only be used "
+                    "in combination with the '--restore-session' option"
+                )
+                sys.exit(1)
+
+            testcases = list(filter(
+                lambda t: report.case(*t)['result'] == 'failure',
+                testcases
+            ))
+            printer.verbose(
+                f'Filtering successful test case(s): '
+                f'{len(testcases)} remaining'
+            )
+
         # Prepare for running
         printer.debug('Building and validating the full test DAG')
         testgraph, skipped_cases = dependencies.build_deps(testcases_all)
@@ -707,11 +735,19 @@ def print_infoline(param, value):
         printer.debug('Full test DAG:')
         printer.debug(dependencies.format_deps(testgraph))
         if len(testcases) != len(testcases_all):
-            testgraph = dependencies.prune_deps(testgraph, testcases)
+            testgraph = dependencies.prune_deps(
+                testgraph, testcases,
+                max_depth=1 if options.restore_session is not None else None
+            )
             printer.debug('Pruned test DAG')
             printer.debug(dependencies.format_deps(testgraph))
+            printer.info(dependencies.format_deps(testgraph))
+            testgraph = report.restore_dangling(testgraph)
 
-        testcases = dependencies.toposort(testgraph)
+        testcases = dependencies.toposort(
+            testgraph,
+            is_subgraph=options.restore_session is not None
+        )
         printer.verbose(f'Final number of test cases: {len(testcases)}')
 
         # Disable hooks
@@ -815,7 +851,7 @@ def print_infoline(param, value):
             session_info['time_start'] = time.strftime(
                 '%FT%T%z', time.localtime(time_start),
             )
-            runner.runall(testcases)
+            runner.runall(testcases, testcases_all)
         finally:
             time_end = time.time()
             session_info['time_end'] = time.strftime(
@@ -856,7 +892,7 @@ def print_infoline(param, value):
                 'session_info': session_info,
                 'runs': run_stats
             }
-            report_file = generate_report_filename(report_file)
+            report_file = runreport.next_report_filename(report_file)
             try:
                 with open(report_file, 'w') as fp:
                     jsonext.dump(json_report, fp, indent=2)
diff --git a/reframe/frontend/dependencies.py b/reframe/frontend/dependencies.py
index dea4b0dde7..cd749937e2 100644
--- a/reframe/frontend/dependencies.py
+++ b/reframe/frontend/dependencies.py
@@ -9,6 +9,7 @@
 
 import collections
 import itertools
+import sys
 
 import reframe as rfm
 import reframe.utility as util
@@ -158,21 +159,24 @@ def validate_deps(graph):
         sources -= visited
 
 
-def prune_deps(graph, testcases):
+def prune_deps(graph, testcases, max_depth=None):
     '''Prune the graph so that it contains only the specified cases and their
-    dependencies.
+    dependencies up to max_depth.
 
     Graph is assumed to by a DAG.
     '''
 
+    max_depth = max_depth or sys.maxsize
     pruned_graph = {}
     for tc in testcases:
         unvisited = [tc]
-        while unvisited:
+        curr_depth = 0
+        while unvisited and curr_depth < max_depth:
             node = unvisited.pop()
             pruned_graph.setdefault(node, util.OrderedSet())
             for adj in graph[node]:
                 pruned_graph[node].add(adj)
+                curr_depth += 1
                 if adj not in pruned_graph:
                     unvisited.append(adj)
 
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 954b7da7b5..903d2e9a35 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -56,19 +56,18 @@ def case(self, check, part, env):
         c, p, e = check.name, part.fullname, env.name
         return self._cases_index.get((c, p, e))
 
-    def restore_deps(self, testcases):
-        '''Restore state of successful dependencies in testcases
+    def restore_dangling(self, graph):
+        '''Restore dangling dependencies in graph from the report data.
 
-        Returns the updated tescases.
+        Returns the updated graph.
         '''
 
-        for tc in testcases:
-            for d in tc.deps:
-                print(d)
-                if self.case(*d)['result'] == 'success':
+        for tc, deps in graph.items():
+            for d in deps:
+                if d not in graph:
                     self._do_restore(d)
 
-        return testcases
+        return graph
 
     def _do_restore(self, testcase):
         dump_file = os.path.join(self.case(*testcase)['stagedir'],

From 152b23d1bdce67be0be0892ef8e3019cf8b215bf Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Tue, 17 Nov 2020 00:16:26 +0100
Subject: [PATCH 13/29] WIP: Generalize rerun option

---
 reframe/frontend/cli.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index e98edcfd90..1654c5b037 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -711,10 +711,14 @@ def print_infoline(param, value):
                 )
                 sys.exit(1)
 
-            testcases = list(filter(
-                lambda t: report.case(*t)['result'] == 'failure',
-                testcases
-            ))
+            def _case_failed(t):
+                rec = report.case(*t)
+                if rec and rec['result'] == 'failure':
+                    return True
+                else:
+                    return False
+
+            testcases = list(filter(_case_failed, testcases))
             printer.verbose(
                 f'Filtering successful test case(s): '
                 f'{len(testcases)} remaining'
@@ -741,7 +745,6 @@ def print_infoline(param, value):
             )
             printer.debug('Pruned test DAG')
             printer.debug(dependencies.format_deps(testgraph))
-            printer.info(dependencies.format_deps(testgraph))
             testgraph = report.restore_dangling(testgraph)
 
         testcases = dependencies.toposort(

From 906777afe27080b630025ce2bdc09ff2d1e6e132 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Wed, 18 Nov 2020 00:08:06 +0100
Subject: [PATCH 14/29] Add support for restoring a testing session

---
 reframe/frontend/cli.py        |  10 +-
 reframe/frontend/runreport.py  |  21 +++-
 reframe/schemas/runreport.json | 183 +++++++++++++++++----------------
 unittests/test_cli.py          |   7 +-
 4 files changed, 125 insertions(+), 96 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 1654c5b037..1753470a30 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -745,7 +745,8 @@ def _case_failed(t):
             )
             printer.debug('Pruned test DAG')
             printer.debug(dependencies.format_deps(testgraph))
-            testgraph = report.restore_dangling(testgraph)
+            if options.restore_session is not None:
+                testgraph, restored_cases = report.restore_dangling(testgraph)
 
         testcases = dependencies.toposort(
             testgraph,
@@ -893,8 +894,13 @@ def _case_failed(t):
             })
             json_report = {
                 'session_info': session_info,
-                'runs': run_stats
+                'runs': run_stats,
+                'restored_cases': []
             }
+            if options.restore_session is not None:
+                for c in restored_cases:
+                    json_report['restored_cases'].append(report.case(*c))
+
             report_file = runreport.next_report_filename(report_file)
             try:
                 with open(report_file, 'w') as fp:
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 903d2e9a35..e9a6f25129 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -31,6 +31,11 @@ def __init__(self, report):
                 c, p, e = tc['name'], tc['system'], tc['environment']
                 self._cases_index[c, p, e] = tc
 
+        # Index also the restored cases
+        for tc in self._report['restored_cases']:
+            c, p, e = tc['name'], tc['system'], tc['environment']
+            self._cases_index[c, p, e] = tc
+
     def __getattr__(self, name):
         return getattr(self._report, name)
 
@@ -62,22 +67,30 @@ def restore_dangling(self, graph):
         Returns the updated graph.
         '''
 
+        restored = []
         for tc, deps in graph.items():
             for d in deps:
                 if d not in graph:
+                    restored.append(d)
                     self._do_restore(d)
 
-        return graph
+        return graph, restored
 
     def _do_restore(self, testcase):
-        dump_file = os.path.join(self.case(*testcase)['stagedir'],
-                                 '.rfm_testcase.json')
+        tc = self.case(*testcase)
+        if tc is None:
+            raise errors.ReframeError(
+                f'could not restore testcase {testcase!r}: '
+                f'not found in the report file'
+            )
+
+        dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
         try:
             with open(dump_file) as fp:
                 jsonext.load(fp, rfm_obj=testcase.check)
         except (OSError, json.JSONDecodeError) as e:
             raise errors.ReframeError(
-                f'could not restore testase {testcase!r}') from e
+                f'could not restore testcase {testcase!r}') from e
 
 
 def next_report_filename(filepatt, new=True):
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index f86f7209e2..720681cc8c 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -2,6 +2,92 @@
     "$schema": "http://json-schema.org/draft-07/schema#",
     "$id": "https://raw.githubusercontent.com/eth-cscs/reframe/master/reframe/schemas/runreport.json",
     "title": "Validation schema for ReFrame's run report",
+    "defs": {
+        "testcase_type": {
+            "type": "object",
+            "properties": {
+                "build_stderr": {"type": ["string", "null"]},
+                "build_stdout": {"type": ["string", "null"]},
+                "description": {"type": "string"},
+                "prefix": {"type": "string"},
+                "filename": {"type": "string"},
+                "environment": {"type": ["string", "null"]},
+                "fail_info": {
+                    "type": ["object", "null"],
+                    "properties": {
+                        "exc_type": {"type": "string"},
+                        "exc_value": {"type": "string"},
+                        "traceback": {
+                            "type": ["array", "null"],
+                            "items": {"type": "string"}
+                        }
+                    },
+                    "required": ["exc_type", "exc_value", "traceback"]
+                },
+                "fail_phase": {"type": ["string", "null"]},
+                "fail_reason": {"type": ["string", "null"]},
+                "fail_severe": {"type": "boolean"},
+                "jobid": {"type": ["string", "null"]},
+                "job_stderr": {"type": ["string", "null"]},
+                "job_stdout": {"type": ["string", "null"]},
+                "name": {"type": "string"},
+                "maintainers": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
+                "nodelist": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
+                "outputdir": {"type": ["string", "null"]},
+                "perfvars": {
+                    "type": ["array", "null"],
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "name": {"type": "string"},
+                            "reference": {
+                                "type": ["number", "null"]
+                            },
+                            "thres_lower": {
+                                "type": ["number", "null"]
+                            },
+                            "thres_upper": {
+                                "type": ["number", "null"]
+                            },
+                            "unit": {"type": ["string", "null"]},
+                            "value": {"type": "number"}
+                        },
+                        "required": [
+                            "name", "reference",
+                            "thres_lower", "thres_upper",
+                            "unit", "value"
+                        ]
+                    }
+                },
+                "result": {
+                    "type": "string",
+                    "enum": ["success", "failure"]
+                },
+                "scheduler": {"type": ["string", "null"]},
+                "stagedir": {"type": ["string", "null"]},
+                "system": {"type": "string"},
+                "tags": {
+                    "type": "array",
+                    "items": {"type": "string"}
+                },
+                "time_compile": {"type": ["number", "null"]},
+                "time_performance": {"type": ["number", "null"]},
+                "time_run": {"type": ["number", "null"]},
+                "time_sanity": {"type": ["number", "null"]},
+                "time_setup": {"type": ["number", "null"]},
+                "time_total": {"type": ["number", "null"]}
+            },
+            "required": [
+                "environment", "name", "result", "system", "filename"
+            ]
+        }
+    },
     "type": "object",
     "properties": {
         "session_info": {
@@ -24,6 +110,10 @@
             },
             "required": ["data_version"]
         },
+        "restored_cases": {
+            "type": "array",
+            "items": {"$ref": "#/defs/testcase_type"}
+        },
         "runs": {
             "type": "array",
             "items": {
@@ -34,95 +124,12 @@
                     "runid": {"type": "number"},
                     "testcases": {
                         "type": "array",
-                        "items": {
-                            "type": "object",
-                            "properties": {
-                                "build_stderr": {"type": ["string", "null"]},
-                                "build_stdout": {"type": ["string", "null"]},
-                                "description": {"type": "string"},
-                                "prefix": {"type": "string"},
-                                "filename": {"type": "string"},
-                                "environment": {"type": ["string", "null"]},
-                                "fail_info": {
-                                    "type": ["object", "null"],
-                                    "properties": {
-                                        "exc_type": {"type": "string"},
-                                        "exc_value": {"type": "string"},
-                                        "traceback": {
-                                            "type": ["array", "null"],
-                                            "items": {"type": "string"}
-                                        }
-                                    },
-                                    "required": ["exc_type", "exc_value", "traceback"]
-                                },
-                                "fail_phase": {"type": ["string", "null"]},
-                                "fail_reason": {"type": ["string", "null"]},
-                                "fail_severe": {"type": "boolean"},
-                                "jobid": {"type": ["string", "null"]},
-                                "job_stderr": {"type": ["string", "null"]},
-                                "job_stdout": {"type": ["string", "null"]},
-                                "name": {"type": "string"},
-                                "maintainers": {
-                                    "type": "array",
-                                    "items": {"type": "string"}
-                                },
-                                "nodelist": {
-                                    "type": "array",
-                                    "items": {"type": "string"}
-                                },
-                                "outputdir": {"type": ["string", "null"]},
-                                "perfvars": {
-                                    "type": ["array", "null"],
-                                    "items": {
-                                        "type": "object",
-                                        "properties": {
-                                            "name": {"type": "string"},
-                                            "reference": {
-                                                "type": ["number", "null"]
-                                            },
-                                            "thres_lower": {
-                                                "type": ["number", "null"]
-                                            },
-                                            "thres_upper": {
-                                                "type": ["number", "null"]
-                                            },
-                                            "unit": {"type": ["string", "null"]},
-                                            "value": {"type": "number"}
-                                        },
-                                        "required": [
-                                            "name", "reference",
-                                            "thres_lower", "thres_upper",
-                                            "unit", "value"
-                                        ]
-                                    }
-                                },
-                                "result": {
-                                    "type": "string",
-                                    "enum": ["success", "failure"]
-                                },
-                                "scheduler": {"type": ["string", "null"]},
-                                "stagedir": {"type": ["string", "null"]},
-                                "system": {"type": "string"},
-                                "tags": {
-                                    "type": "array",
-                                    "items": {"type": "string"}
-                                },
-                                "time_compile": {"type": ["number", "null"]},
-                                "time_performance": {"type": ["number", "null"]},
-                                "time_run": {"type": ["number", "null"]},
-                                "time_sanity": {"type": ["number", "null"]},
-                                "time_setup": {"type": ["number", "null"]},
-                                "time_total": {"type": ["number", "null"]}
-                            },
-                            "required": [
-                                "environment", "name", "result", "system", "filename"
-                            ]
-                        },
-                        "required": [
-                            "num_cases", "num_failures", "runid", "testcases"
-                        ]
+                        "items": {"$ref": "#/defs/testcase_type"}
                     }
-                }
+                },
+                "required": [
+                    "num_cases", "num_failures", "runid", "testcases"
+                ]
             }
         }
     },
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index 9d317355f1..bc36bfbdfc 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -146,13 +146,16 @@ def test_check_success(run_reframe, tmp_path):
     assert os.path.exists(tmp_path / 'report.json')
 
 
-def test_check_retry_failed(run_reframe, tmp_path):
+def test_check_restore_session_failed(run_reframe, tmp_path):
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
     )
     returncode, stdout, _ = run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
-        more_options=['--retry-failed', f'{tmp_path}/report.json']
+        more_options=[
+            f'--restore-session={tmp_path}/report.json',
+            f'--failed'
+        ]
     )
     with open(f'{tmp_path}/report.json') as fp:
         report = json.load(fp)

From 2faf16c475488b3a96b8acf546ac78bf7eb69e6c Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Sun, 22 Nov 2020 20:45:11 +0100
Subject: [PATCH 15/29] Fix execution policies when retrying restored cases

---
 reframe/frontend/cli.py                |  4 +++-
 reframe/frontend/executors/__init__.py |  8 +++-----
 reframe/frontend/executors/policies.py | 24 ++++++++++++++++--------
 3 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 1753470a30..bdf470dc93 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -747,6 +747,8 @@ def _case_failed(t):
             printer.debug(dependencies.format_deps(testgraph))
             if options.restore_session is not None:
                 testgraph, restored_cases = report.restore_dangling(testgraph)
+        else:
+            restored_cases = []
 
         testcases = dependencies.toposort(
             testgraph,
@@ -855,7 +857,7 @@ def _case_failed(t):
             session_info['time_start'] = time.strftime(
                 '%FT%T%z', time.localtime(time_start),
             )
-            runner.runall(testcases, testcases_all)
+            runner.runall(testcases, restored_cases)
         finally:
             time_end = time.time()
             session_info['time_end'] = time.strftime(
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index e2193dbb36..43e5193e8b 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -379,7 +379,7 @@ def policy(self):
     def stats(self):
         return self._stats
 
-    def runall(self, testcases, testcases_og=None):
+    def runall(self, testcases, restored_cases=None):
         num_checks = len({tc.check.name for tc in testcases})
         self._printer.separator('short double line',
                                 'Running %d check(s)' % num_checks)
@@ -388,10 +388,8 @@ def runall(self, testcases, testcases_og=None):
         try:
             self._runall(testcases)
             if self._max_retries:
-                if testcases_og:
-                    self._retry_failed(testcases_og)
-                else:
-                    self._retry_failed(testcases)
+                restored_cases = restored_cases or []
+                self._retry_failed(testcases + restored_cases)
 
         finally:
             # Print the summary line
diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py
index 248e60e873..9634554791 100644
--- a/reframe/frontend/executors/policies.py
+++ b/reframe/frontend/executors/policies.py
@@ -94,7 +94,9 @@ def runcase(self, case):
         self.stats.add_task(task)
         try:
             # Do not run test if any of its dependencies has failed
-            if any(self._task_index[c].failed for c in case.deps):
+            # NOTE: Restored dependencies are not in the task_index
+            if any(self._task_index[c].failed
+                   for c in case.deps if c in self._task_index):
                 raise TaskDependencyError('dependencies failed')
 
             partname = task.testcase.partition.fullname
@@ -181,9 +183,11 @@ def on_task_success(self, task):
                                          'total'])
         getlogger().verbose(f'==> {timings}')
 
-        # update reference count of dependencies
+        # Update reference count of dependencies
         for c in task.testcase.deps:
-            self._task_index[c].ref_count -= 1
+            # NOTE: Restored dependencies are not in the task_index
+            if c in self._task_index:
+                self._task_index[c].ref_count -= 1
 
         _cleanup_all(self._retired_tasks, not self.keep_stage_files)
 
@@ -237,12 +241,14 @@ def _remove_from_running(self, task):
             pass
 
     def deps_failed(self, task):
-        return any(self._task_index[c].failed for c in task.testcase.deps)
+        # NOTE: Restored dependencies are not in the task_index
+        return any(self._task_index[c].failed
+                   for c in task.testcase.deps if c in self._task_index)
 
     def deps_succeeded(self, task):
+        # NOTE: Restored dependencies are not in the task_index
         return all(self._task_index[c].succeeded
-                   for c in task.testcase.deps
-                   if c in self._task_index)
+                   for c in task.testcase.deps if c in self._task_index)
 
     def on_task_setup(self, task):
         partname = task.check.current_partition.fullname
@@ -273,9 +279,11 @@ def on_task_success(self, task):
         self.printer.status('OK', msg, just='right')
         getlogger().verbose(f'==> timings: {task.pipeline_timings_all()}')
 
-        # update reference count of dependencies
+        # Update reference count of dependencies
         for c in task.testcase.deps:
-            self._task_index[c].ref_count -= 1
+            # NOTE: Restored dependencies are not in the task_index
+            if c in self._task_index:
+                self._task_index[c].ref_count -= 1
 
         self._retired_tasks.append(task)
 

From d9b163d65722714ddde78f23c4639be2dd0f67a9 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Sun, 22 Nov 2020 21:04:37 +0100
Subject: [PATCH 16/29] Fix setting of restored_cases

---
 reframe/frontend/cli.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index bdf470dc93..774ad6ebba 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -738,6 +738,8 @@ def _case_failed(t):
         dependencies.validate_deps(testgraph)
         printer.debug('Full test DAG:')
         printer.debug(dependencies.format_deps(testgraph))
+
+        restored_cases = []
         if len(testcases) != len(testcases_all):
             testgraph = dependencies.prune_deps(
                 testgraph, testcases,
@@ -747,8 +749,6 @@ def _case_failed(t):
             printer.debug(dependencies.format_deps(testgraph))
             if options.restore_session is not None:
                 testgraph, restored_cases = report.restore_dangling(testgraph)
-        else:
-            restored_cases = []
 
         testcases = dependencies.toposort(
             testgraph,

From de828865b760b08e5b56e5245cc7eca968dd39a3 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Tue, 24 Nov 2020 00:13:24 +0100
Subject: [PATCH 17/29] Expand unit tests

---
 reframe/core/config.py        |  3 ++
 reframe/frontend/cli.py       | 16 ++++++++-
 reframe/frontend/runreport.py | 11 +++++--
 unittests/test_cli.py         | 62 +++++++++++++++++++++++++----------
 unittests/test_config.py      |  1 +
 5 files changed, 73 insertions(+), 20 deletions(-)

diff --git a/reframe/core/config.py b/reframe/core/config.py
index 185b7f996e..e25889c355 100644
--- a/reframe/core/config.py
+++ b/reframe/core/config.py
@@ -111,6 +111,9 @@ def add_sticky_option(self, option, value):
     def remove_sticky_option(self, option):
         self._sticky_options.pop(option, None)
 
+    def is_sticky_option(self, option):
+        return option in self._sticky_options
+
     @_normalize_syntax({'.*/.*modules$': normalize_module_list})
     def get(self, option, default=None):
         '''Retrieve value of option.
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 774ad6ebba..0a82756d7a 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -592,6 +592,19 @@ def main():
         report = runreport.load_report(filename)
         check_search_path = list(report.slice('filename', unique=True))
         check_search_recursive = False
+
+        # If `-c` or `-R` are passed explicitly outside the configuration
+        # file, override the values set from the report file
+        if site_config.is_sticky_option('general/check_search_path'):
+            check_search_path = site_config.get(
+                'general/0/check_search_path'
+            )
+
+        if site_config.is_sticky_option('general/check_search_recursive'):
+            check_search_recursive = site_config.get(
+                'general/0/check_search_recursive'
+            )
+
     else:
         check_search_recursive = site_config.get(
             'general/0/check_search_recursive'
@@ -602,7 +615,8 @@ def main():
         load_path=check_search_path,
         recurse=check_search_recursive,
         ignore_conflicts=site_config.get(
-            'general/0/ignore_check_conflicts')
+            'general/0/ignore_check_conflicts'
+        )
     )
 
     def print_infoline(param, value):
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index e9a6f25129..0ff600a35c 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -36,6 +36,9 @@ def __init__(self, report):
             c, p, e = tc['name'], tc['system'], tc['environment']
             self._cases_index[c, p, e] = tc
 
+    def __getitem__(self, key):
+        return self._report[key]
+
     def __getattr__(self, name):
         return getattr(self._report, name)
 
@@ -51,10 +54,14 @@ def slice(self, prop, when=None, unique=False):
                 continue
 
             if when is None:
-                returned.add(val)
+                if unique:
+                    returned.add(val)
+
                 yield val
             elif tc[when[0]] == when[1]:
-                returned.add(val)
+                if unique:
+                    returned.add(val)
+
                 yield val
 
     def case(self, check, part, env):
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index bc36bfbdfc..f4322e3e62 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -15,6 +15,7 @@
 
 import reframe.core.config as config
 import reframe.core.environments as env
+import reframe.frontend.runreport as runreport
 import reframe.core.logging as logging
 import reframe.core.runtime as rt
 import unittests.fixtures as fixtures
@@ -147,31 +148,58 @@ def test_check_success(run_reframe, tmp_path):
 
 
 def test_check_restore_session_failed(run_reframe, tmp_path):
-    returncode, stdout, _ = run_reframe(
+    run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
     )
     returncode, stdout, _ = run_reframe(
+        checkpath=[],
+        more_options=[
+            f'--restore-session={tmp_path}/report.json', '--failed'
+        ]
+    )
+    report = runreport.load_report(f'{tmp_path}/report.json')
+    assert set(report.slice('name', when=('fail_phase', 'sanity'))) == {'T2'}
+    assert set(report.slice('name',
+                            when=('fail_phase', 'startup'))) == {'T7', 'T9'}
+    assert set(report.slice('name', when=('fail_phase', 'setup'))) == {'T8'}
+    assert report['runs'][-1]['num_cases'] == 4
+
+    restored = {r['name'] for r in report['restored_cases']}
+    assert restored == {'T1', 'T6'}
+
+
+def test_check_restore_session_succeeded_test(run_reframe, tmp_path):
+    run_reframe(
         checkpath=['unittests/resources/checks_unlisted/deps_complex.py'],
+        more_options=['--keep-stage-files']
+    )
+    returncode, stdout, _ = run_reframe(
+        checkpath=[],
         more_options=[
-            f'--restore-session={tmp_path}/report.json',
-            f'--failed'
+            f'--restore-session={tmp_path}/report.json', '-n', 'T1'
         ]
     )
-    with open(f'{tmp_path}/report.json') as fp:
-        report = json.load(fp)
+    report = runreport.load_report(f'{tmp_path}/report.json')
+    assert report['runs'][-1]['num_cases'] == 1
+    assert report['runs'][-1]['testcases'][0]['name'] == 'T1'
+
+    restored = {r['name'] for r in report['restored_cases']}
+    assert restored == {'T4', 'T5'}
 
-    report_summary = {
-        t['name']: t['fail_phase']
-        for run in report['runs']
-        for t in run['testcases']
-    }
-    assert report_summary['T2'] == 'sanity'
-    assert report_summary['T7'] == 'startup'
-    assert report_summary['T8'] == 'setup'
-    assert report_summary['T9'] == 'startup'
 
-    assert all(i not in report_summary.keys()
-               for i in ['T0', 'T1', 'T3', 'T4', 'T5', 'T6'])
+def test_check_restore_session_check_search_path(run_reframe, tmp_path):
+    run_reframe(
+        checkpath=['unittests/resources/checks_unlisted/deps_complex.py']
+    )
+    returncode, stdout, _ = run_reframe(
+        checkpath=[f'{tmp_path}/foo'],
+        more_options=[
+            f'--restore-session={tmp_path}/report.json', '-n', 'T1', '-R'
+        ],
+        action='list'
+    )
+    assert returncode == 0
+    assert 'Found 0 check(s)' in stdout
 
 
 def test_check_success_force_local(run_reframe, tmp_path):
@@ -184,7 +212,7 @@ def test_check_success_force_local(run_reframe, tmp_path):
 
 
 def test_report_file_with_sessionid(run_reframe, tmp_path):
-    returncode, stdout, _ = run_reframe(
+    returncode, _, _ = run_reframe(
         more_options=[
             f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}'
         ]
diff --git a/unittests/test_config.py b/unittests/test_config.py
index 6b0fe9f223..b1d1a6b2e7 100644
--- a/unittests/test_config.py
+++ b/unittests/test_config.py
@@ -315,6 +315,7 @@ def test_sticky_options():
     site_config.select_subconfig('testsys:login')
     site_config.add_sticky_option('environments/cc', 'clang')
     site_config.add_sticky_option('modes/options', ['foo'])
+    assert site_config.is_sticky_option('modes/options')
     assert site_config.get('environments/@PrgEnv-gnu/cc') == 'clang'
     assert site_config.get('environments/@PrgEnv-cray/cc') == 'clang'
     assert site_config.get('environments/@PrgEnv-cray/cxx') == 'CC'

From b817ad7232b542afd9fe258a85f7a059322dd8a1 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Wed, 25 Nov 2020 01:05:37 +0100
Subject: [PATCH 18/29] JSON (de)serialization of ReFrame objects

- This brings along full restore capability for ReFrame tests.
---
 reframe/core/deferrable.py             |  3 --
 reframe/core/environments.py           |  3 +-
 reframe/core/pipeline.py               | 15 +------
 reframe/core/schedulers/__init__.py    |  3 +-
 reframe/core/systems.py                |  5 ++-
 reframe/frontend/executors/__init__.py | 36 +++++++--------
 reframe/frontend/runreport.py          | 21 +--------
 reframe/utility/jsonext.py             | 62 +++++++++++++++++++++-----
 unittests/test_utility.py              | 57 +++++++++++++++++++++++
 9 files changed, 135 insertions(+), 70 deletions(-)

diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py
index dcc937fcad..c7be9e01f0 100644
--- a/reframe/core/deferrable.py
+++ b/reframe/core/deferrable.py
@@ -75,9 +75,6 @@ def __iter__(self):
         '''Evaluate the deferred expression and iterate over the result.'''
         return iter(self.evaluate())
 
-    def __rfm_json_encode__(self):
-        return self.evaluate()
-
     # Overload Python operators to be able to defer any expression
     #
     # NOTE: In the following we are not using `self` for denoting the first
diff --git a/reframe/core/environments.py b/reframe/core/environments.py
index c11170a19d..64c8904592 100644
--- a/reframe/core/environments.py
+++ b/reframe/core/environments.py
@@ -8,6 +8,7 @@
 
 import reframe.core.fields as fields
 import reframe.utility as util
+import reframe.utility.jsonext as jsonext
 import reframe.utility.typecheck as typ
 
 
@@ -26,7 +27,7 @@ def normalize_module_list(modules):
     return ret
 
 
-class Environment:
+class Environment(jsonext.JSONSerializable):
     '''This class abstracts away an environment to run regression tests.
 
     It is simply a collection of modules to be loaded and environment variables
diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 15c7c4af48..25cc70cd98 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -26,6 +26,7 @@
 import reframe.core.logging as logging
 import reframe.core.runtime as rt
 import reframe.utility as util
+import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 import reframe.utility.sanity as sn
 import reframe.utility.typecheck as typ
@@ -125,7 +126,7 @@ def _wrapped(*args, **kwargs):
     return _wrapped
 
 
-class RegressionTest(metaclass=RegressionTestMeta):
+class RegressionTest(jsonext.JSONSerializable, metaclass=RegressionTestMeta):
     '''Base class for regression tests.
 
     All regression tests must eventually inherit from this class.
@@ -838,18 +839,6 @@ def _rfm_init(self, name=None, prefix=None):
             # Just an empty environment
             self._cdt_environ = env.Environment('__rfm_cdt_environ')
 
-    def __rfm_json_encode__(self):
-        return {
-            'modules': self.modules,
-            'variables': self.variables,
-            'stagedir': self.stagedir,
-        }
-
-    def __rfm_json_restore__(self, jsonobj):
-        self.modules = jsonobj['modules']
-        self.variables = jsonobj['variables']
-        self._stagedir = jsonobj['stagedir']
-
     # Export read-only views to interesting fields
     @property
     def current_environ(self):
diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py
index 36daa0620f..5809d12ec5 100644
--- a/reframe/core/schedulers/__init__.py
+++ b/reframe/core/schedulers/__init__.py
@@ -13,6 +13,7 @@
 import reframe.core.fields as fields
 import reframe.core.runtime as runtime
 import reframe.core.shell as shell
+import reframe.utility.jsonext as jsonext
 import reframe.utility.typecheck as typ
 from reframe.core.exceptions import JobError, JobNotStartedError
 from reframe.core.launchers import JobLauncher
@@ -111,7 +112,7 @@ def log(self, message, level=DEBUG2):
         getlogger().log(level, f'[S] {self.registered_name}: {message}')
 
 
-class Job:
+class Job(jsonext.JSONSerializable):
     '''A job descriptor.
 
     A job descriptor is created by the framework after the "setup" phase and
diff --git a/reframe/core/systems.py b/reframe/core/systems.py
index 4b2bb6e3da..56bf5bda9e 100644
--- a/reframe/core/systems.py
+++ b/reframe/core/systems.py
@@ -7,13 +7,14 @@
 import re
 
 import reframe.utility as utility
+import reframe.utility.jsonext as jsonext
 from reframe.core.backends import (getlauncher, getscheduler)
 from reframe.core.logging import getlogger
 from reframe.core.modules import ModulesSystem
 from reframe.core.environments import (Environment, ProgEnvironment)
 
 
-class SystemPartition:
+class SystemPartition(jsonext.JSONSerializable):
     '''A representation of a system partition inside ReFrame.
 
     .. warning::
@@ -237,7 +238,7 @@ def __str__(self):
         return json.dumps(self.json(), indent=2)
 
 
-class System:
+class System(jsonext.JSONSerializable):
     '''A representation of a system inside ReFrame.
 
     .. warning::
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 43e5193e8b..3ec04143e9 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -33,12 +33,12 @@ class TestCase:
     '''
 
     def __init__(self, check, partition, environ):
-        self.__check_orig = check
-        self.__check = copy.deepcopy(check)
-        self.__partition = copy.deepcopy(partition)
-        self.__environ = copy.deepcopy(environ)
-        self.__check._case = weakref.ref(self)
-        self.__deps = []
+        self._check_orig = check
+        self._check = copy.deepcopy(check)
+        self._partition = copy.deepcopy(partition)
+        self._environ = copy.deepcopy(environ)
+        self._check._case = weakref.ref(self)
+        self._deps = []
 
         # Incoming dependencies
         self.in_degree = 0
@@ -46,7 +46,7 @@ def __init__(self, check, partition, environ):
     def __iter__(self):
         # Allow unpacking a test case with a single liner:
         #       c, p, e = case
-        return iter([self.__check, self.__partition, self.__environ])
+        return iter([self._check, self._partition, self._environ])
 
     def __hash__(self):
         return (hash(self.check.name) ^
@@ -67,19 +67,19 @@ def __repr__(self):
 
     @property
     def check(self):
-        return self.__check
+        return self._check
 
     @property
     def partition(self):
-        return self.__partition
+        return self._partition
 
     @property
     def environ(self):
-        return self.__environ
+        return self._environ
 
     @property
     def deps(self):
-        return self.__deps
+        return self._deps
 
     @property
     def num_dependents(self):
@@ -87,7 +87,7 @@ def num_dependents(self):
 
     def clone(self):
         # Return a fresh clone, i.e., one based on the original check
-        return TestCase(self.__check_orig, self.__partition, self.__environ)
+        return TestCase(self._check_orig, self._partition, self._environ)
 
 
 def generate_testcases(checks,
@@ -292,13 +292,11 @@ def performance(self):
 
     def finalize(self):
         try:
-            json_check = os.path.join(self.check.stagedir,
-                                      '.rfm_testcase.json')
-            with open(json_check, 'w') as fp:
-                jsonext.dump(self.check, fp)
-        except OSError:
-            self._printer.warning(f'check {RegressionTask(t).check.name} '
-                                  f'can not be dumped')
+            jsonfile = os.path.join(self.check.stagedir, '.rfm_testcase.json')
+            with open(jsonfile, 'w') as fp:
+                jsonext.dump(self.check, fp, indent=2)
+        except OSError as e:
+            self._printer.warning(f'could not dump test case {self.case}: {e}')
 
         self._current_stage = 'finalize'
         self._notify_listeners('on_task_success')
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 0ff600a35c..29dfff5f5c 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -10,6 +10,7 @@
 
 import reframe as rfm
 import reframe.core.exceptions as errors
+import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 
 
@@ -94,7 +95,7 @@ def _do_restore(self, testcase):
         dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
         try:
             with open(dump_file) as fp:
-                jsonext.load(fp, rfm_obj=testcase.check)
+                testcase._check = jsonext.load(fp)
         except (OSError, json.JSONDecodeError) as e:
             raise errors.ReframeError(
                 f'could not restore testcase {testcase!r}') from e
@@ -148,21 +149,3 @@ def load_report(filename):
         )
 
     return _RunReport(report)
-
-
-def restore(testcases, retry_report, printer):
-    stagedirs = {}
-    for run in retry_report['runs']:
-        for t in run['testcases']:
-            idx = (t['name'], t['system'], t['environment'])
-            stagedirs[idx] = t['stagedir']
-
-    for i, t in enumerate(testcases):
-        idx = (t.check.name, t.partition.fullname, t.environ.name)
-        try:
-            with open(os.path.join(stagedirs[idx],
-                                   '.rfm_testcase.json')) as f:
-                jsonext.load(f, rfm_obj=RegressionTask(t).check)
-        except (OSError, json.JSONDecodeError):
-            printer.warning(f'check {RegressionTask(t).check.name} '
-                            f'can not be restored')
diff --git a/reframe/utility/jsonext.py b/reframe/utility/jsonext.py
index d4ab700c89..cbec30b81e 100644
--- a/reframe/utility/jsonext.py
+++ b/reframe/utility/jsonext.py
@@ -7,6 +7,18 @@
 import json
 import traceback
 
+import reframe.utility as util
+
+
+class JSONSerializable:
+    def __rfm_json_encode__(self):
+        ret = {
+            '__rfm_class__': type(self).__qualname__,
+            '__rfm_file__': inspect.getfile(type(self))
+        }
+        ret.update(self.__dict__)
+        return ret
+
 
 class _ReframeJsonEncoder(json.JSONEncoder):
     def default(self, obj):
@@ -20,10 +32,16 @@ def default(self, obj):
         if isinstance(obj, BaseException):
             return str(obj)
 
+        if isinstance(obj, set):
+            return list(obj)
+
         if inspect.istraceback(obj):
             return traceback.format_tb(obj)
 
-        return json.JSONEncoder.default(self, obj)
+        try:
+            return json.JSONEncoder.default(self, obj)
+        except TypeError:
+            return None
 
 
 def dump(obj, fp, **kwargs):
@@ -36,23 +54,43 @@ def dumps(obj, **kwargs):
     return json.dumps(obj, **kwargs)
 
 
+def _object_hook(json):
+    filename = json.pop('__rfm_file__', None)
+    typename = json.pop('__rfm_class__', None)
+    if filename is None or typename is None:
+        return json
+
+    mod = util.import_module_from_file(filename)
+    cls = getattr(mod, typename)
+    obj = cls.__new__(cls)
+    obj.__dict__.update(json)
+    return obj
+
+
 class _ReframeJsonDecoder(json.JSONDecoder):
     def __init__(self, *args, **kwargs):
-        if 'rfm_obj' in kwargs:
-            self.rfm_obj = kwargs['rfm_obj']
-            del kwargs['rfm_obj']
-
-        json.JSONDecoder.__init__(self, object_hook=self.object_hook,
-                                  *args, **kwargs)
+        self.__target = kwargs.pop('_target', None)
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
     def object_hook(self, obj):
-        if 'modules' in obj:
-            self.rfm_obj.__rfm_json_restore__(obj)
-            return self.rfm_obj
+        target_typename = type(self.__target).__qualname__
+        if '__rfm_class__' not in obj:
+            return obj
+
+        if target_typename != obj['__rfm_class__']:
+            return obj
 
-        return obj
+        if hasattr(self.__target, '__rfm_json_decode__'):
+            return self.__target.__rfm_json_decode__(obj)
+        else:
+            return obj
 
 
 def load(fp, **kwargs):
-    kwargs['cls'] = _ReframeJsonDecoder
+    kwargs['object_hook'] = _object_hook
     return json.load(fp, **kwargs)
+
+
+def loads(s, **kwargs):
+    kwargs['object_hook'] = _object_hook
+    return json.loads(s, **kwargs)
diff --git a/unittests/test_utility.py b/unittests/test_utility.py
index 1975049b9e..cf8447fe1b 100644
--- a/unittests/test_utility.py
+++ b/unittests/test_utility.py
@@ -1485,6 +1485,63 @@ def test_jsonext_dumps():
                                               separators=(',', ':'))
 
 
+# Classes to test JSON deserialization
+
+class _D(jsonext.JSONSerializable):
+    def __init__(self):
+        self.a = 2
+        self.b = 'bar'
+
+    def __eq__(self, other):
+        if not isinstance(other, _D):
+            return NotImplemented
+
+        return self.a == other.a and self.b == other.b
+
+
+class _Z(_D):
+    pass
+
+
+class _C(jsonext.JSONSerializable):
+    def __init__(self, x, y):
+        self.x = x
+        self.y = y
+        self.z = None
+
+    def __eq__(self, other):
+        if not isinstance(other, _C):
+            return NotImplemented
+
+        return (self.x == other.x and
+                self.y == other.y and
+                self.z == other.z)
+
+
+def test_jsonext_load(tmp_path):
+    c = _C(1, 'foo')
+    c.x += 1
+    c.y = 'foobar'
+    c.z = _Z()
+    c.z.a += 1
+    c.z.b = 'barfoo'
+
+    json_dump = tmp_path / 'test.json'
+    with open(json_dump, 'w') as fp:
+        jsonext.dump(c, fp, indent=2)
+
+    with open(json_dump, 'r') as fp:
+        c_restored = jsonext.load(fp)
+
+    assert c == c_restored
+    assert c is not c_restored
+
+    # Do the same with dumps() and loads()
+    c_restored = jsonext.loads(jsonext.dumps(c))
+    assert c == c_restored
+    assert c is not c_restored
+
+
 def test_attr_validator():
     class C:
         def __init__(self):

From 0f9336df47fefa5dc4214552c951b10bb24c0464 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Thu, 26 Nov 2020 01:14:44 +0100
Subject: [PATCH 19/29] Suppress exceptions when dumping deferrables

---
 reframe/core/deferrable.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/reframe/core/deferrable.py b/reframe/core/deferrable.py
index c7be9e01f0..8d820f8b31 100644
--- a/reframe/core/deferrable.py
+++ b/reframe/core/deferrable.py
@@ -75,6 +75,12 @@ def __iter__(self):
         '''Evaluate the deferred expression and iterate over the result.'''
         return iter(self.evaluate())
 
+    def __rfm_json_encode__(self):
+        try:
+            return self.evaluate()
+        except BaseException:
+            return None
+
     # Overload Python operators to be able to defer any expression
     #
     # NOTE: In the following we are not using `self` for denoting the first

From d7e68af57c4ba11a1bcd5b41fdb68a9e4d8afd02 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Thu, 26 Nov 2020 08:50:40 +0100
Subject: [PATCH 20/29] Add hook to allow objects to customize the JSON decode
 process.

---
 reframe/utility/jsonext.py | 3 +++
 unittests/test_utility.py  | 9 ++++++++-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/reframe/utility/jsonext.py b/reframe/utility/jsonext.py
index cbec30b81e..dd3c84fd91 100644
--- a/reframe/utility/jsonext.py
+++ b/reframe/utility/jsonext.py
@@ -64,6 +64,9 @@ def _object_hook(json):
     cls = getattr(mod, typename)
     obj = cls.__new__(cls)
     obj.__dict__.update(json)
+    if hasattr(obj, '__rfm_json_decode__'):
+        obj.__rfm_json_decode__(json)
+
     return obj
 
 
diff --git a/unittests/test_utility.py b/unittests/test_utility.py
index cf8447fe1b..0f536eee49 100644
--- a/unittests/test_utility.py
+++ b/unittests/test_utility.py
@@ -1508,6 +1508,12 @@ def __init__(self, x, y):
         self.x = x
         self.y = y
         self.z = None
+        self.w = {1, 2}
+
+    def __rfm_json_decode__(self, json):
+        # Sets are converted to lists when encoding, we need to manually
+        # change them back to sets
+        self.w = set(json['w'])
 
     def __eq__(self, other):
         if not isinstance(other, _C):
@@ -1515,7 +1521,8 @@ def __eq__(self, other):
 
         return (self.x == other.x and
                 self.y == other.y and
-                self.z == other.z)
+                self.z == other.z and
+                self.w == other.w)
 
 
 def test_jsonext_load(tmp_path):

From dc6b7f22b312b1297e375e50f83183cd0dd098a8 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Thu, 26 Nov 2020 19:44:23 +0100
Subject: [PATCH 21/29] More unit tests

---
 reframe/frontend/cli.py         |   2 +-
 reframe/frontend/runreport.py   |   6 +-
 reframe/schemas/runreport.json  |   2 +-
 unittests/resources/settings.py |   2 +-
 unittests/test_policies.py      | 136 ++++++++++++++++++++++++++------
 5 files changed, 120 insertions(+), 28 deletions(-)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 0a82756d7a..91bfdce522 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -626,7 +626,7 @@ def print_infoline(param, value):
     session_info = {
         'cmdline': ' '.join(sys.argv),
         'config_file': rt.site_config.filename,
-        'data_version': '1.1',
+        'data_version': runreport.DATA_VERSION,
         'hostname': socket.gethostname(),
         'prefix_output': rt.output_prefix,
         'prefix_stage': rt.stage_prefix,
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 29dfff5f5c..2bdb1928c5 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -14,8 +14,8 @@
 import reframe.utility.jsonext as jsonext
 
 
+DATA_VERSION = '1.1'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
-_DATA_VERSION = '1.1'
 
 
 class _RunReport:
@@ -142,10 +142,10 @@ def load_report(filename):
 
     # Check if the report data is compatible
     data_ver = report['session_info']['data_version']
-    if data_ver != _DATA_VERSION:
+    if data_ver != DATA_VERSION:
         raise errors.ReframeError(
             f'incompatible report data versions: '
-            f'found {data_ver!r}, required {_DATA_VERSION!r}'
+            f'found {data_ver!r}, required {DATA_VERSION!r}'
         )
 
     return _RunReport(report)
diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json
index 720681cc8c..2ca3bc2e0a 100644
--- a/reframe/schemas/runreport.json
+++ b/reframe/schemas/runreport.json
@@ -133,5 +133,5 @@
             }
         }
     },
-    "required": ["runs"]
+    "required": ["restored_cases", "runs", "session_info"]
 }
diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py
index b38eafa3a2..7680458d0e 100644
--- a/unittests/resources/settings.py
+++ b/unittests/resources/settings.py
@@ -19,7 +19,7 @@
                     'descr': 'Login nodes',
                     'scheduler': 'local',
                     'launcher': 'local',
-                    'environs': ['builtin-gcc']
+                    'environs': ['builtin']
                 }
             ]
         },
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index ba3c1c3ee4..7c76992dff 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -17,11 +17,13 @@
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
 import reframe.frontend.executors.policies as policies
+import reframe.frontend.runreport as runreport
 import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.exceptions import (AbortTaskError,
                                      JobNotStartedError,
+                                     ReframeError,
                                      ReframeForceExitError,
                                      TaskDependencyError)
 from reframe.frontend.loader import RegressionCheckLoader
@@ -42,6 +44,25 @@
 )
 
 
+# NOTE: We could move this to utility
+class timer:
+    '''Context manager for timing'''
+
+    def __init__(self):
+        self._time_start = None
+        self._time_end = None
+
+    def __enter__(self):
+        self._time_start = time.time()
+        return self
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._time_end = time.time()
+
+    def timestamps(self):
+        return self._time_start, self._time_end
+
+
 @pytest.fixture
 def temp_runtime(tmp_path):
     def _temp_runtime(site_config, system=None, options={}):
@@ -127,26 +148,12 @@ def _validate_runreport(report):
     jsonschema.validate(json.loads(report), schema)
 
 
-def test_runall(make_runner, make_cases, common_exec_ctx):
-    runner = make_runner()
-    time_start = time.time()
-    runner.runall(make_cases())
-    time_end = time.time()
-    assert 9 == runner.stats.num_cases()
-    assert_runall(runner)
-    assert 5 == len(runner.stats.failures())
-    assert 2 == num_failures_stage(runner, 'setup')
-    assert 1 == num_failures_stage(runner, 'sanity')
-    assert 1 == num_failures_stage(runner, 'performance')
-    assert 1 == num_failures_stage(runner, 'cleanup')
-
-    # Create a run report and validate it
-    run_stats = runner.stats.json()
-    report = {
+def _generate_runreport(run_stats, time_start, time_end):
+    return {
         'session_info': {
             'cmdline': ' '.join(sys.argv),
             'config_file': rt.runtime().site_config.filename,
-            'data_version': '1.0',
+            'data_version': runreport.DATA_VERSION,
             'hostname': socket.gethostname(),
             'num_cases': run_stats[0]['num_cases'],
             'num_failures': run_stats[-1]['num_failures'],
@@ -163,16 +170,55 @@ def test_runall(make_runner, make_cases, common_exec_ctx):
             'version': osext.reframe_version(),
             'workdir': os.getcwd()
         },
+        'restored_cases': [],
         'runs': run_stats
     }
 
+
+def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
+    runner = make_runner()
+    with timer() as tm:
+        runner.runall(make_cases())
+
+    assert 9 == runner.stats.num_cases()
+    assert_runall(runner)
+    assert 5 == len(runner.stats.failures())
+    assert 2 == num_failures_stage(runner, 'setup')
+    assert 1 == num_failures_stage(runner, 'sanity')
+    assert 1 == num_failures_stage(runner, 'performance')
+    assert 1 == num_failures_stage(runner, 'cleanup')
+
+    # Create a run report and validate it
+    report = _generate_runreport(runner.stats.json(), *tm.timestamps())
+
     # We dump the report first, in order to get any object conversions right
-    final_report = None
-    with io.StringIO() as fp:
-        jsonext.dump(report, fp, indent=2)
-        final_report = fp.getvalue()
+    report_file = tmp_path / 'report.json'
+    with open(report_file, 'w') as fp:
+        jsonext.dump(report, fp)
+
+    # Read and validate the report using the runreport module
+    runreport.load_report(report_file)
+
+    # Try to load a non-existent report
+    with pytest.raises(ReframeError, match='failed to load report file'):
+        runreport.load_report(tmp_path / 'does_not_exist.json')
 
-    _validate_runreport(final_report)
+    # Generate an invalid JSON
+    with open(tmp_path / 'invalid.json', 'w') as fp:
+        jsonext.dump(report, fp)
+        fp.write('invalid')
+
+    with pytest.raises(ReframeError, match=r'is not a valid JSON file'):
+        runreport.load_report(tmp_path / 'invalid.json')
+
+    # Generate a report with an incorrect data version
+    report['session_info']['data_version'] = 'not-a-version'
+    with open(tmp_path / 'invalid-version.json', 'w') as fp:
+        jsonext.dump(report, fp)
+
+    with pytest.raises(ReframeError,
+                       match=r'incompatible report data versions'):
+        runreport.load_report(tmp_path / 'invalid-version.json')
 
 
 def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx):
@@ -699,3 +745,49 @@ def test_compile_fail_reschedule_busy_loop(async_runner, make_cases,
     assert num_checks == stats.num_cases()
     assert_runall(runner)
     assert num_checks == len(stats.failures())
+
+
+@pytest.fixture
+def report_file(make_runner, dep_cases, common_exec_ctx, tmp_path):
+    runner = make_runner()
+    runner.policy.keep_stage_files = True
+    with timer() as tm:
+        runner.runall(dep_cases)
+
+    report = _generate_runreport(runner.stats.json(), *tm.timestamps())
+    filename = tmp_path / 'report.json'
+    with open(filename, 'w') as fp:
+        jsonext.dump(report, fp)
+
+    return filename
+
+
+def test_restore_session(report_file, make_runner,
+                         dep_cases, common_exec_ctx, tmp_path):
+    # Select a single test to run and create the pruned graph
+    selected = [tc for tc in dep_cases if tc.check.name == 'T1']
+    testgraph = dependencies.prune_deps(
+        dependencies.build_deps(dep_cases)[0], selected, max_depth=1
+    )
+
+    # Restore the required test cases
+    report = runreport.load_report(report_file)
+    testgraph, restored_cases = report.restore_dangling(testgraph)
+
+    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
+
+    # Run the selected test cases
+    runner = make_runner()
+    with timer() as tm:
+        runner.runall(selected, restored_cases)
+
+    new_report = _generate_runreport(runner.stats.json(), *tm.timestamps())
+    assert new_report['runs'][0]['num_cases'] == 1
+    assert new_report['runs'][0]['testcases'][0]['name'] == 'T1'
+
+    # Remove the test case dump file and retry
+    os.remove(tmp_path / 'stage' / 'generic' / 'default' /
+              'builtin' / 'T4' / '.rfm_testcase.json')
+
+    with pytest.raises(ReframeError, match=r'could not restore testcase'):
+        report.restore_dangling(testgraph)

From 004aac764ee04610d668d4e11249f366f66cecfa Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Thu, 26 Nov 2020 20:22:21 +0100
Subject: [PATCH 22/29] Remove `builtin-gcc` from test settings

Also:

- Properly decode `tags` in `RegressionTest`
---
 reframe/core/pipeline.py        |  4 ++++
 unittests/resources/settings.py | 12 +++---------
 unittests/test_cli.py           |  8 ++++----
 unittests/test_config.py        |  8 ++++----
 unittests/test_pipeline.py      |  6 +++---
 unittests/test_policies.py      |  2 +-
 6 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py
index 25cc70cd98..7f8deb6b1a 100644
--- a/reframe/core/pipeline.py
+++ b/reframe/core/pipeline.py
@@ -1753,6 +1753,10 @@ def __eq__(self, other):
     def __hash__(self):
         return hash(self.name)
 
+    def __rfm_json_decode__(self, json):
+        # 'tags' are decoded as list, so we convert them to a set
+        self.tags = set(json['tags'])
+
 
 class RunOnlyRegressionTest(RegressionTest, special=True):
     '''Base class for run-only regression tests.
diff --git a/unittests/resources/settings.py b/unittests/resources/settings.py
index 7680458d0e..e4b5389754 100644
--- a/unittests/resources/settings.py
+++ b/unittests/resources/settings.py
@@ -36,7 +36,7 @@
                     'name': 'login',
                     'scheduler': 'local',
                     'launcher': 'local',
-                    'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'],
+                    'environs': ['PrgEnv-cray', 'PrgEnv-gnu'],
                     'descr': 'Login nodes'
                 },
                 {
@@ -61,7 +61,7 @@
                             ]
                         }
                     ],
-                    'environs': ['PrgEnv-gnu', 'builtin-gcc'],
+                    'environs': ['PrgEnv-gnu', 'builtin'],
                     'max_jobs': 10
                 }
             ]
@@ -110,12 +110,6 @@
             'cxx': '',
             'ftn': ''
         },
-        {
-            'name': 'builtin-gcc',
-            'cc': 'gcc',
-            'cxx': 'g++',
-            'ftn': 'gfortran'
-        },
         {
             'name': 'e0',
             'modules': ['m0']
@@ -134,7 +128,7 @@
             'name': 'unittest',
             'options': [
                 '-c unittests/resources/checks/hellocheck.py',
-                '-p builtin-gcc',
+                '-p builtin',
                 '--force-local'
             ]
         }
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index f4322e3e62..11a3c391ad 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -58,7 +58,7 @@ def perflogdir(tmp_path):
 def run_reframe(tmp_path, perflogdir):
     def _run_reframe(system='generic:default',
                      checkpath=['unittests/resources/checks/hellocheck.py'],
-                     environs=['builtin-gcc'],
+                     environs=['builtin'],
                      local=True,
                      action='run',
                      more_options=None,
@@ -301,7 +301,7 @@ def test_check_sanity_failure(run_reframe, tmp_path):
     assert returncode != 0
     assert os.path.exists(
         tmp_path / 'stage' / 'generic' / 'default' /
-        'builtin-gcc' / 'SanityFailureCheck'
+        'builtin' / 'SanityFailureCheck'
     )
 
 
@@ -314,7 +314,7 @@ def test_dont_restage(run_reframe, tmp_path):
     # Place a random file in the test's stage directory and rerun with
     # `--dont-restage` and `--max-retries`
     stagedir = (tmp_path / 'stage' / 'generic' / 'default' /
-                'builtin-gcc' / 'SanityFailureCheck')
+                'builtin' / 'SanityFailureCheck')
     (stagedir / 'foobar').touch()
     returncode, stdout, stderr = run_reframe(
         checkpath=['unittests/resources/checks/frontend_checks.py'],
@@ -361,7 +361,7 @@ def test_performance_check_failure(run_reframe, tmp_path, perflogdir):
     assert returncode != 0
     assert os.path.exists(
         tmp_path / 'stage' / 'generic' / 'default' /
-        'builtin-gcc' / 'PerformanceFailureCheck'
+        'builtin' / 'PerformanceFailureCheck'
     )
     assert os.path.exists(perflogdir / 'generic' /
                           'default' / 'PerformanceFailureCheck.log')
diff --git a/unittests/test_config.py b/unittests/test_config.py
index b1d1a6b2e7..c0d483ff12 100644
--- a/unittests/test_config.py
+++ b/unittests/test_config.py
@@ -247,7 +247,7 @@ def test_select_subconfig():
     assert site_config.get('systems/0/partitions/0/scheduler') == 'local'
     assert site_config.get('systems/0/partitions/0/launcher') == 'local'
     assert (site_config.get('systems/0/partitions/0/environs') ==
-            ['PrgEnv-cray', 'PrgEnv-gnu', 'builtin-gcc'])
+            ['PrgEnv-cray', 'PrgEnv-gnu'])
     assert site_config.get('systems/0/partitions/0/descr') == 'Login nodes'
     assert site_config.get('systems/0/partitions/0/resources') == []
     assert site_config.get('systems/0/partitions/0/access') == []
@@ -255,7 +255,7 @@ def test_select_subconfig():
     assert site_config.get('systems/0/partitions/0/modules') == []
     assert site_config.get('systems/0/partitions/0/variables') == []
     assert site_config.get('systems/0/partitions/0/max_jobs') == 8
-    assert len(site_config['environments']) == 6
+    assert len(site_config['environments']) == 5
     assert site_config.get('environments/@PrgEnv-gnu/cc') == 'gcc'
     assert site_config.get('environments/0/cxx') == 'g++'
     assert site_config.get('environments/@PrgEnv-cray/cc') == 'cc'
@@ -269,7 +269,7 @@ def test_select_subconfig():
     assert site_config.get('systems/0/partitions/@gpu/scheduler') == 'slurm'
     assert site_config.get('systems/0/partitions/0/launcher') == 'srun'
     assert (site_config.get('systems/0/partitions/0/environs') ==
-            ['PrgEnv-gnu', 'builtin-gcc'])
+            ['PrgEnv-gnu', 'builtin'])
     assert site_config.get('systems/0/partitions/0/descr') == 'GPU partition'
     assert len(site_config.get('systems/0/partitions/0/resources')) == 2
     assert (site_config.get('systems/0/partitions/0/resources/@gpu/name') ==
@@ -280,7 +280,7 @@ def test_select_subconfig():
     assert (site_config.get('systems/0/partitions/0/variables') ==
             [['FOO_GPU', 'yes']])
     assert site_config.get('systems/0/partitions/0/max_jobs') == 10
-    assert len(site_config['environments']) == 6
+    assert len(site_config['environments']) == 5
     assert site_config.get('environments/@PrgEnv-gnu/cc') == 'cc'
     assert site_config.get('environments/0/cxx') == 'CC'
     assert site_config.get('general/0/check_search_path') == ['c:d']
diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py
index d83d79144f..9432fc7ca2 100644
--- a/unittests/test_pipeline.py
+++ b/unittests/test_pipeline.py
@@ -73,7 +73,7 @@ def hellotest():
 @pytest.fixture
 def local_exec_ctx(generic_system):
     partition = fixtures.partition_by_name('default')
-    environ = fixtures.environment_by_name('builtin-gcc', partition)
+    environ = fixtures.environment_by_name('builtin', partition)
     yield partition, environ
 
 
@@ -461,7 +461,7 @@ def set_resources(self):
 
     test = MyTest()
     partition = fixtures.partition_by_name('gpu')
-    environ = partition.environment('builtin-gcc')
+    environ = partition.environment('builtin')
     _run(test, partition, environ)
     expected_job_options = {'--gres=gpu:2',
                             '#DW jobdw capacity=100GB',
@@ -852,7 +852,7 @@ def _run_sanity(test, *exec_ctx, skip_perf=False):
 @pytest.fixture
 def dummy_gpu_exec_ctx(testsys_system):
     partition = fixtures.partition_by_name('gpu')
-    environ = fixtures.environment_by_name('builtin-gcc', partition)
+    environ = fixtures.environment_by_name('builtin', partition)
     yield partition, environ
 
 
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index 7c76992dff..b99a4d0831 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -296,7 +296,7 @@ def test_force_local_execution(make_runner, make_cases, testsys_exec_ctx):
     runner = make_runner()
     runner.policy.force_local = True
     test = HelloTest()
-    test.valid_prog_environs = ['builtin-gcc']
+    test.valid_prog_environs = ['builtin']
 
     runner.runall(make_cases([test]))
     assert_runall(runner)

From b9ffdc75117c9a1ff04c3167212bdf1d538d353d Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Thu, 26 Nov 2020 21:13:02 +0100
Subject: [PATCH 23/29] Better data version control for run report

---
 reframe/frontend/runreport.py | 10 ++++++----
 reframe/utility/versioning.py | 16 ++++++++++++++++
 unittests/test_policies.py    |  2 +-
 unittests/test_versioning.py  |  7 ++++++-
 4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index 2bdb1928c5..d2c12a3700 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -12,9 +12,10 @@
 import reframe.core.exceptions as errors
 import reframe.utility as util
 import reframe.utility.jsonext as jsonext
+from reframe.utility.versioning import Version
 
 
-DATA_VERSION = '1.1'
+DATA_VERSION = '1.2'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
 
@@ -141,11 +142,12 @@ def load_report(filename):
         raise errors.ReframeError(f'invalid report {filename!r}') from e
 
     # Check if the report data is compatible
-    data_ver = report['session_info']['data_version']
-    if data_ver != DATA_VERSION:
+    found_ver = Version(report['session_info']['data_version'])
+    required_ver = Version(DATA_VERSION)
+    if found_ver.major != required_ver.major or found_ver < required_ver:
         raise errors.ReframeError(
             f'incompatible report data versions: '
-            f'found {data_ver!r}, required {DATA_VERSION!r}'
+            f'found {found_ver}, required >= {required_ver}'
         )
 
     return _RunReport(report)
diff --git a/reframe/utility/versioning.py b/reframe/utility/versioning.py
index 782fa898ff..ea64136230 100644
--- a/reframe/utility/versioning.py
+++ b/reframe/utility/versioning.py
@@ -32,6 +32,22 @@ def __init__(self, version):
         except ValueError:
             raise ValueError('invalid version string: %s' % version) from None
 
+    @property
+    def major(self):
+        return self._major
+
+    @property
+    def minor(self):
+        return self._minor
+
+    @property
+    def patch_level(self):
+        return self._patch_level
+
+    @property
+    def dev_number(self):
+        return self._dev_number
+
     def _value(self):
         return 10000*self._major + 100*self._minor + self._patch_level
 
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index b99a4d0831..c561e342e5 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -212,7 +212,7 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path):
         runreport.load_report(tmp_path / 'invalid.json')
 
     # Generate a report with an incorrect data version
-    report['session_info']['data_version'] = 'not-a-version'
+    report['session_info']['data_version'] = '10.0'
     with open(tmp_path / 'invalid-version.json', 'w') as fp:
         jsonext.dump(report, fp)
 
diff --git a/unittests/test_versioning.py b/unittests/test_versioning.py
index d8f64ec305..fa6ce9fc92 100644
--- a/unittests/test_versioning.py
+++ b/unittests/test_versioning.py
@@ -14,7 +14,12 @@ def test_version_format():
     Version('1.2.3')
     Version('1.2-dev0')
     Version('1.2-dev5')
-    Version('1.2.3-dev2')
+    v = Version('1.2.3-dev2')
+    assert v.major == 1
+    assert v.minor == 2
+    assert v.patch_level == 3
+    assert v.dev_number == 2
+
     with pytest.raises(ValueError):
         Version(None)
 

From f85e7eebc14e25bce965207637dca25a989f615f Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Fri, 27 Nov 2020 11:13:47 +0100
Subject: [PATCH 24/29] Update documentation

---
 docs/manpage.rst | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/docs/manpage.rst b/docs/manpage.rst
index 8f01c4403c..723dd9106d 100644
--- a/docs/manpage.rst
+++ b/docs/manpage.rst
@@ -62,7 +62,8 @@ After all tests in the search path have been loaded, they are first filtered by
 Any test that is not valid for the current system, it will be filtered out.
 The current system is either auto-selected or explicitly specified with the :option:`--system` option.
 Tests can be filtered by different attributes and there are specific command line options for achieving this.
-
+A common characteristic of all test filtering options is that if a test is selected, then all its dependencies will be selected, too, regardless if they match the filtering criteria or not.
+This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` depends on ``T3``, then selecting ``T1`` would also select ``T2`` and ``T3``.
 
 .. option:: -t, --tag=TAG
 
@@ -116,6 +117,15 @@ Tests can be filtered by different attributes and there are specific command lin
    Tests may or may not make use of it.
 
 
+.. option:: --failed
+
+   Select only the failed test cases for a previous run.
+   This option can only be used in combination with the :option:`--restore-session`.
+   To rerun the failed cases from the last run, you can use ``reframe --restore-session --failed -r``.
+
+   .. versionadded:: 3.4
+
+
 .. option:: --skip-system-check
 
    Do not filter tests against the selected system.
@@ -196,7 +206,7 @@ Options controlling ReFrame output
 
    This option can also be set using the :envvar:`RFM_STAGE_DIR` environment variable or the :js:attr:`stagedir` system configuration parameter.
 
-.. option:: --timestamp[=TIMEFMT]
+.. option:: --timestamp [TIMEFMT]
 
    Append a timestamp to the output and stage directory prefixes.
    ``TIMEFMT`` can be any valid :manpage:`strftime(3)` time format.
@@ -312,6 +322,25 @@ Options controlling ReFrame execution
    .. versionadded:: 3.2
 
 
+.. option:: --restore-session [REPORT]
+
+   Restore a testing session that has run previously.
+   ``REPORT`` is a run report file generated by ReFrame.
+   If ``REPORT`` is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option).
+   If passed alone, this option will simply rerun all the test cases that have run previously based on the report file data.
+   It is more useful to combine this option with any of the `test filtering <#test-filtering>`__ options, in which case only the selected test cases will be executed.
+   The difference in test selection process when using this option is that the dependencies of the selected tests will not be selected for execution, as they would normally, but they will be restored.
+   For example, if test ``T1`` depends on ``T2`` and ``T2`` depends on ``T3``, then running ``reframe -n T1 -r`` would cause both ``T2`` and ``T3`` to run.
+   However, by doing ``reframe -n T1 --restore-session -r``, only ``T1`` would run and its immediate dependence ``T2`` will be restored.
+   This is useful when you have deep test dependencies or some of the tests in the dependency chain are very time consuming.
+
+   .. note::
+      In order for a test case to be restored, its stage directory must be present.
+      This is not a problem when rerunning a failed case, since the stage directories of its dependencies are automatically kept, but if you want to rerun a successful test case, you should make sure to have run with the :option:`--keep-stage-files` option.
+
+   .. versionadded:: 3.4
+
+
 ----------------------------------
 Options controlling job submission
 ----------------------------------
@@ -463,7 +492,7 @@ Miscellaneous options
 
    This option can also be set using the :envvar:`RFM_CONFIG_FILE` environment variable.
 
-.. option:: --show-config[=PARAM]
+.. option:: --show-config [PARAM]
 
    Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit.
    The parameter value is printed in JSON format.

From 63fda8302d85047051d7c471b46762a622d7f476 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Tue, 1 Dec 2020 11:11:13 +0100
Subject: [PATCH 25/29] Address PR comments

---
 reframe/frontend/cli.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 483b209909..761388e916 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -600,11 +600,20 @@ def main():
         # If `-c` or `-R` are passed explicitly outside the configuration
         # file, override the values set from the report file
         if site_config.is_sticky_option('general/check_search_path'):
+            printer.warning(
+                'Ignoring check search path set in the report file: '
+                'search path set explicitly in the command-line or '
+                'the environment'
+            )
             check_search_path = site_config.get(
                 'general/0/check_search_path'
             )
 
         if site_config.is_sticky_option('general/check_search_recursive'):
+            printer.warning(
+                'Ignoring check search recursive option from the report file: '
+                'option set explicitly in the command-line or the environment'
+            )
             check_search_recursive = site_config.get(
                 'general/0/check_search_recursive'
             )

From 7356cec94280acb88dee34836a782d276b785f22 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Tue, 1 Dec 2020 12:02:22 +0100
Subject: [PATCH 26/29] Remove unused imports

---
 reframe/core/config.py                 | 3 ---
 reframe/core/systems.py                | 1 -
 reframe/frontend/cli.py                | 3 ---
 reframe/frontend/dependencies.py       | 1 -
 reframe/frontend/executors/__init__.py | 2 --
 reframe/frontend/executors/policies.py | 3 +--
 reframe/frontend/runreport.py          | 1 -
 unittests/test_cli.py                  | 1 -
 unittests/test_config.py               | 1 -
 unittests/test_policies.py             | 4 ----
 unittests/test_utility.py              | 2 --
 unittests/test_versioning.py           | 1 -
 12 files changed, 1 insertion(+), 22 deletions(-)

diff --git a/reframe/core/config.py b/reframe/core/config.py
index e25889c355..c50e060743 100644
--- a/reframe/core/config.py
+++ b/reframe/core/config.py
@@ -15,11 +15,8 @@
 import tempfile
 
 import reframe
-import reframe.core.fields as fields
 import reframe.core.settings as settings
 import reframe.utility as util
-import reframe.utility.osext as osext
-import reframe.utility.typecheck as types
 from reframe.core.environments import normalize_module_list
 from reframe.core.exceptions import ConfigError, ReframeFatalError
 from reframe.core.logging import getlogger
diff --git a/reframe/core/systems.py b/reframe/core/systems.py
index 56bf5bda9e..537d9716df 100644
--- a/reframe/core/systems.py
+++ b/reframe/core/systems.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import json
-import re
 
 import reframe.utility as utility
 import reframe.utility.jsonext as jsonext
diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py
index 761388e916..402e106917 100644
--- a/reframe/frontend/cli.py
+++ b/reframe/frontend/cli.py
@@ -6,7 +6,6 @@
 import inspect
 import itertools
 import json
-import jsonschema
 import os
 import re
 import shlex
@@ -34,8 +33,6 @@
 from reframe.frontend.executors.policies import (SerialExecutionPolicy,
                                                  AsynchronousExecutionPolicy)
 from reframe.frontend.executors import Runner, generate_testcases
-from reframe.frontend.executors import (RegressionTask, Runner,
-                                        generate_testcases)
 
 
 def format_check(check, check_deps, detailed=False):
diff --git a/reframe/frontend/dependencies.py b/reframe/frontend/dependencies.py
index cd749937e2..96d54bf236 100644
--- a/reframe/frontend/dependencies.py
+++ b/reframe/frontend/dependencies.py
@@ -11,7 +11,6 @@
 import itertools
 import sys
 
-import reframe as rfm
 import reframe.utility as util
 from reframe.core.exceptions import DependencyError
 from reframe.core.logging import getlogger
diff --git a/reframe/frontend/executors/__init__.py b/reframe/frontend/executors/__init__.py
index 3ec04143e9..88e4c68c6a 100644
--- a/reframe/frontend/executors/__init__.py
+++ b/reframe/frontend/executors/__init__.py
@@ -5,14 +5,12 @@
 
 import abc
 import copy
-import json
 import os
 import signal
 import sys
 import time
 import weakref
 
-import reframe.core.environments as env
 import reframe.core.logging as logging
 import reframe.core.runtime as runtime
 import reframe.frontend.dependencies as dependencies
diff --git a/reframe/frontend/executors/policies.py b/reframe/frontend/executors/policies.py
index 9634554791..5dab71103f 100644
--- a/reframe/frontend/executors/policies.py
+++ b/reframe/frontend/executors/policies.py
@@ -6,12 +6,11 @@
 import contextlib
 import functools
 import itertools
-import math
 import sys
 import time
 
 from reframe.core.exceptions import (TaskDependencyError, TaskExit)
-from reframe.core.logging import (getlogger, VERBOSE)
+from reframe.core.logging import getlogger
 from reframe.frontend.executors import (ExecutionPolicy, RegressionTask,
                                         TaskEventListener, ABORT_REASONS)
 
diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py
index d2c12a3700..127076758b 100644
--- a/reframe/frontend/runreport.py
+++ b/reframe/frontend/runreport.py
@@ -10,7 +10,6 @@
 
 import reframe as rfm
 import reframe.core.exceptions as errors
-import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 from reframe.utility.versioning import Version
 
diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index beb108b38e..a76280681a 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -6,7 +6,6 @@
 import contextlib
 import io
 import itertools
-import json
 import os
 import pytest
 import re
diff --git a/unittests/test_config.py b/unittests/test_config.py
index c0d483ff12..d856f021d6 100644
--- a/unittests/test_config.py
+++ b/unittests/test_config.py
@@ -4,7 +4,6 @@
 # SPDX-License-Identifier: BSD-3-Clause
 
 import json
-import os
 import pytest
 
 import reframe.core.config as config
diff --git a/unittests/test_policies.py b/unittests/test_policies.py
index c561e342e5..8a563cb38d 100644
--- a/unittests/test_policies.py
+++ b/unittests/test_policies.py
@@ -3,7 +3,6 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import io
 import json
 import jsonschema
 import os
@@ -12,17 +11,14 @@
 import sys
 import time
 
-import reframe
 import reframe.core.runtime as rt
 import reframe.frontend.dependencies as dependencies
 import reframe.frontend.executors as executors
 import reframe.frontend.executors.policies as policies
 import reframe.frontend.runreport as runreport
-import reframe.utility as util
 import reframe.utility.jsonext as jsonext
 import reframe.utility.osext as osext
 from reframe.core.exceptions import (AbortTaskError,
-                                     JobNotStartedError,
                                      ReframeError,
                                      ReframeForceExitError,
                                      TaskDependencyError)
diff --git a/unittests/test_utility.py b/unittests/test_utility.py
index a6cd32a82d..2d2424a972 100644
--- a/unittests/test_utility.py
+++ b/unittests/test_utility.py
@@ -6,12 +6,10 @@
 import os
 import pytest
 import random
-import shutil
 import sys
 import time
 
 import reframe
-import reframe.core.fields as fields
 import reframe.core.runtime as rt
 import reframe.utility as util
 import reframe.utility.jsonext as jsonext
diff --git a/unittests/test_versioning.py b/unittests/test_versioning.py
index fa6ce9fc92..88ba7cf020 100644
--- a/unittests/test_versioning.py
+++ b/unittests/test_versioning.py
@@ -5,7 +5,6 @@
 
 import pytest
 
-from reframe.frontend.loader import RegressionCheckLoader
 from reframe.utility.versioning import Version, VersionValidator
 
 

From 259445f85c9d50eef90535a7ea5f6ad583d54355 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 7 Dec 2020 19:05:48 +0100
Subject: [PATCH 27/29] Address PR comments

---
 unittests/test_cli.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unittests/test_cli.py b/unittests/test_cli.py
index a76280681a..25f85ec0b5 100644
--- a/unittests/test_cli.py
+++ b/unittests/test_cli.py
@@ -209,7 +209,7 @@ def test_check_success_force_local(run_reframe, tmp_path):
 
 
 def test_report_file_with_sessionid(run_reframe, tmp_path):
-    returncode, _, _ = run_reframe(
+    returncode, *_ = run_reframe(
         more_options=[
             f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}'
         ]

From c511a53ba04f711452e388e89f40b53586d73417 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 7 Dec 2020 19:18:09 +0100
Subject: [PATCH 28/29] Remove unused imports

---
 unittests/test_schedulers.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py
index 7d5ecc710e..8099465791 100644
--- a/unittests/test_schedulers.py
+++ b/unittests/test_schedulers.py
@@ -3,15 +3,12 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
-import abc
-import functools
 import os
 import pytest
 import re
 import signal
 import socket
 import time
-from datetime import datetime, timedelta
 
 import reframe.core.runtime as rt
 import unittests.fixtures as fixtures
@@ -20,7 +17,6 @@
 from reframe.core.exceptions import (
     JobError, JobNotStartedError, JobSchedulerError
 )
-from reframe.core.launchers.local import LocalLauncher
 from reframe.core.schedulers import Job
 from reframe.core.schedulers.slurm import _SlurmNode, _create_nodes
 

From b8eacc441850d8f636e70958d1c5c0fce2c1a745 Mon Sep 17 00:00:00 2001
From: Vasileios Karakasis <karakasis@cscs.ch>
Date: Mon, 7 Dec 2020 19:28:25 +0100
Subject: [PATCH 29/29] Replace use of `datetime` with `time.time()`

---
 unittests/test_schedulers.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/unittests/test_schedulers.py b/unittests/test_schedulers.py
index 8099465791..310b88f94a 100644
--- a/unittests/test_schedulers.py
+++ b/unittests/test_schedulers.py
@@ -283,13 +283,13 @@ def test_submit(make_job, exec_ctx):
 def test_submit_timelimit(minimal_job, local_only):
     minimal_job.time_limit = '2s'
     prepare_job(minimal_job, 'sleep 10')
-    t_job = datetime.now()
+    t_job = time.time()
     minimal_job.submit()
     assert minimal_job.jobid is not None
     minimal_job.wait()
-    t_job = datetime.now() - t_job
-    assert t_job.total_seconds() >= 2
-    assert t_job.total_seconds() < 3
+    t_job = time.time() - t_job
+    assert t_job >= 2
+    assert t_job < 3
     with open(minimal_job.stdout) as fp:
         assert re.search('postrun', fp.read()) is None
 
@@ -319,7 +319,7 @@ def test_submit_job_array(make_job, slurm_only, exec_ctx):
 def test_cancel(make_job, exec_ctx):
     minimal_job = make_job(sched_access=exec_ctx.access)
     prepare_job(minimal_job, 'sleep 30')
-    t_job = datetime.now()
+    t_job = time.time()
     minimal_job.submit()
     minimal_job.cancel()
 
@@ -331,9 +331,9 @@ def test_cancel(make_job, exec_ctx):
     time.sleep(0.01)
 
     minimal_job.wait()
-    t_job = datetime.now() - t_job
+    t_job = time.time() - t_job
     assert minimal_job.finished()
-    assert t_job.total_seconds() < 30
+    assert t_job < 30
 
     # Additional scheduler-specific checks
     sched_name = minimal_job.scheduler.registered_name