diff --git a/docs/manpage.rst b/docs/manpage.rst index dd265eef61..69a5d0123d 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -358,11 +358,11 @@ Options controlling ReFrame execution .. versionadded:: 3.2 -.. option:: --restore-session [REPORT] +.. option:: --restore-session [REPORT1[,REPORT2,...]] Restore a testing session that has run previously. - ``REPORT`` is a run report file generated by ReFrame. - If ``REPORT`` is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option). + ``REPORT1`` etc. are a run report files generated by ReFrame. + If a report is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option). If passed alone, this option will simply rerun all the test cases that have run previously based on the report file data. It is more useful to combine this option with any of the `test filtering <#test-filtering>`__ options, in which case only the selected test cases will be executed. The difference in test selection process when using this option is that the dependencies of the selected tests will not be selected for execution, as they would normally, but they will be restored. @@ -370,12 +370,18 @@ Options controlling ReFrame execution However, by doing ``reframe -n T1 --restore-session -r``, only ``T1`` would run and its immediate dependence ``T2`` will be restored. This is useful when you have deep test dependencies or some of the tests in the dependency chain are very time consuming. + Multiple reports may be passed as a comma-separated list. + ReFrame will try to restore any required test case by looking it up in each report sequentially. + If it cannot find it, it will issue an error and exit. + .. note:: In order for a test case to be restored, its stage directory must be present. This is not a problem when rerunning a failed case, since the stage directories of its dependencies are automatically kept, but if you want to rerun a successful test case, you should make sure to have run with the :option:`--keep-stage-files` option. .. versionadded:: 3.4 + .. versionchanged:: 3.6.1 + Multiple report files are now accepted. ---------------------------------- Options controlling job submission diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index dd1cccf927..c39d92e1a0 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -25,11 +25,13 @@ def rfm_command(testcase): else: config_opt = '' - report_file = f'rfm-report-{testcase.level}.json' + report_file = f'{testcase.check.name}-report.json' if testcase.level: - restore_file = f'rfm-report-{testcase.level - 1}.json' + restore_files = ','.join( + f'{t.check.name}-report.json' for t in tc.deps + ) else: - restore_file = None + restore_files = None return ' '.join([ program, @@ -37,7 +39,7 @@ def rfm_command(testcase): f'{" ".join("-c " + c for c in checkpath)}', f'-R' if recurse else '', f'--report-file={report_file}', - f'--restore-session={restore_file}' if restore_file else '', + f'--restore-session={restore_files}' if restore_files else '', '-n', testcase.check.name, '-r' ]) @@ -54,7 +56,7 @@ def rfm_command(testcase): 'stage': f'rfm-stage-{tc.level}', 'script': [rfm_command(tc)], 'artifacts': { - 'paths': [f'rfm-report-{tc.level}.json'] + 'paths': [f'{tc.check.name}-report.json'] }, 'needs': [t.check.name for t in tc.deps] } diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 81c2d2a787..07df20a1df 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -624,16 +624,16 @@ def main(): # Setup the check loader if options.restore_session is not None: - # We need to load the failed checks only from a report + # We need to load the failed checks only from a list of reports if options.restore_session: - filename = options.restore_session + filenames = options.restore_session.split(',') else: - filename = runreport.next_report_filename( + filenames = [runreport.next_report_filename( osext.expandvars(site_config.get('general/0/report_file')), new=False - ) + )] - report = runreport.load_report(filename) + report = runreport.load_report(*filenames) check_search_path = list(report.slice('filename', unique=True)) check_search_recursive = False @@ -817,6 +817,8 @@ def _case_failed(t): printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: testgraph, restored_cases = report.restore_dangling(testgraph) + print(dependencies.format_deps(testgraph)) + print(restored_cases) testcases = dependencies.toposort( testgraph, diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 1457dbfde5..5e981d2edd 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -24,6 +24,7 @@ class _RunReport: def __init__(self, report): self._report = report + self._fallbacks = [] # fallback reports # Index all runs by test case; if a test case has run multiple times, # only the last time will be indexed @@ -44,6 +45,9 @@ def __getitem__(self, key): def __getattr__(self, name): return getattr(self._report, name) + def add_fallback(self, report): + self._fallbacks.append(report) + def slice(self, prop, when=None, unique=False): '''Slice the report on property ``prop``.''' @@ -68,7 +72,15 @@ def slice(self, prop, when=None, unique=False): def case(self, check, part, env): c, p, e = check.name, part.fullname, env.name - return self._cases_index.get((c, p, e)) + ret = self._cases_index.get((c, p, e)) + if ret is None: + # Look up the case in the fallback reports + for rpt in self._fallbacks: + ret = rpt._cases_index.get((c, p, e)) + if ret is not None: + break + + return ret def restore_dangling(self, graph): '''Restore dangling dependencies in graph from the report data. @@ -90,7 +102,7 @@ def _do_restore(self, testcase): if tc is None: raise errors.ReframeError( f'could not restore testcase {testcase!r}: ' - f'not found in the report file' + f'not found in the report files' ) dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json') @@ -121,7 +133,7 @@ def next_report_filename(filepatt, new=True): return filepatt.format(sessionid=new_id) -def load_report(filename): +def _load_report(filename): try: with open(filename) as fp: report = json.load(fp) @@ -155,6 +167,17 @@ def load_report(filename): return _RunReport(report) +def load_report(*filenames): + primary = filenames[0] + rpt = _load_report(primary) + + # Add fallback reports + for f in filenames[1:]: + rpt.add_fallback(_load_report(f)) + + return rpt + + def junit_xml_report(json_report): '''Generate a JUnit report from a standard ReFrame JSON report.''' diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 52f462e85e..9b7c2988a5 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -853,6 +853,28 @@ def test_restore_session(report_file, make_runner, assert new_report['runs'][0]['num_cases'] == 1 assert new_report['runs'][0]['testcases'][0]['name'] == 'T1' + # Generate an empty report and load it as primary with the original report + # as a fallback, in order to test if the dependencies are still resolved + # correctly + empty_report = tmp_path / 'empty.json' + + with open(empty_report, 'w') as fp: + empty_run = [ + { + 'num_cases': 0, + 'num_failures': 0, + 'num_aborted': 0, + 'num_skipped': 0, + 'runid': 0, + 'testcases': [] + } + ] + jsonext.dump(_generate_runreport(empty_run, *tm.timestamps()), fp) + + report2 = runreport.load_report(empty_report, report_file) + restored_cases = report2.restore_dangling(testgraph)[1] + assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} + # Remove the test case dump file and retry os.remove(tmp_path / 'stage' / 'generic' / 'default' / 'builtin' / 'T4' / '.rfm_testcase.json')