From 12cb15d27e534fca305aea048f990c18edc2e03c Mon Sep 17 00:00:00 2001 From: "G. Ryan Sablosky" Date: Thu, 25 Mar 2021 11:25:02 -0400 Subject: [PATCH 1/6] read in runs from multiple reports --- reframe/frontend/cli.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index c6a14459bd..3c68ddf44d 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -595,14 +595,25 @@ def main(): if options.restore_session is not None: # We need to load the failed checks only from a report if options.restore_session: - filename = options.restore_session + filenames = options.restore_session.split(',') else: - filename = runreport.next_report_filename( + filenames = [runreport.next_report_filename( osext.expandvars(site_config.get('general/0/report_file')), new=False - ) + )] + + + reports = [] + for filename in filenames: + report = runreport.load_report(filename) + reports.append(report) + + # pop the last report, merge in the other reports + # this is almost certainly not the right way to go about this + report = reports.pop() + for rpt in reports: + report._cases_index.update(rpt._cases_index) - report = runreport.load_report(filename) check_search_path = list(report.slice('filename', unique=True)) check_search_recursive = False From e9471de7e792dcc7ae9af6bfaf4acaedc544ac79 Mon Sep 17 00:00:00 2001 From: "G. Ryan Sablosky" Date: Thu, 25 Mar 2021 14:27:37 -0400 Subject: [PATCH 2/6] Generate per-task reports --- reframe/frontend/ci.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index dd1cccf927..afa2d1c81a 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -25,11 +25,11 @@ def rfm_command(testcase): else: config_opt = '' - report_file = f'rfm-report-{testcase.level}.json' + report_file = f'{testcase.check.name}-report.json' if testcase.level: - restore_file = f'rfm-report-{testcase.level - 1}.json' + restore_files = ','.join([f'{t.check.name}-report.json' for t in tc.deps]) else: - restore_file = None + restore_files = None return ' '.join([ program, @@ -37,7 +37,7 @@ def rfm_command(testcase): f'{" ".join("-c " + c for c in checkpath)}', f'-R' if recurse else '', f'--report-file={report_file}', - f'--restore-session={restore_file}' if restore_file else '', + f'--restore-session={restore_files}' if restore_files else '', '-n', testcase.check.name, '-r' ]) @@ -54,7 +54,7 @@ def rfm_command(testcase): 'stage': f'rfm-stage-{tc.level}', 'script': [rfm_command(tc)], 'artifacts': { - 'paths': [f'rfm-report-{tc.level}.json'] + 'paths': [f'{tc.check.name}-report.json'] }, 'needs': [t.check.name for t in tc.deps] } From c6ddac2a24ae2b2722b976ba77f95f08b5d2ef13 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 14 May 2021 00:44:15 +0200 Subject: [PATCH 3/6] Use fallback reports --- reframe/frontend/cli.py | 17 ++++------------- reframe/frontend/runreport.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 3c68ddf44d..ca37673d8d 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -593,7 +593,7 @@ def main(): # Setup the check loader if options.restore_session is not None: - # We need to load the failed checks only from a report + # We need to load the failed checks only from a list of reports if options.restore_session: filenames = options.restore_session.split(',') else: @@ -602,18 +602,7 @@ def main(): new=False )] - - reports = [] - for filename in filenames: - report = runreport.load_report(filename) - reports.append(report) - - # pop the last report, merge in the other reports - # this is almost certainly not the right way to go about this - report = reports.pop() - for rpt in reports: - report._cases_index.update(rpt._cases_index) - + report = runreport.load_report(*filenames) check_search_path = list(report.slice('filename', unique=True)) check_search_recursive = False @@ -797,6 +786,8 @@ def _case_failed(t): printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: testgraph, restored_cases = report.restore_dangling(testgraph) + print(dependencies.format_deps(testgraph)) + print(restored_cases) testcases = dependencies.toposort( testgraph, diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 1cedb843ec..a63bfa8469 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -23,6 +23,7 @@ class _RunReport: def __init__(self, report): self._report = report + self._fallbacks = [] # fallback reports # Index all runs by test case; if a test case has run multiple times, # only the last time will be indexed @@ -43,6 +44,9 @@ def __getitem__(self, key): def __getattr__(self, name): return getattr(self._report, name) + def add_fallback(self, report): + self._fallbacks.append(report) + def slice(self, prop, when=None, unique=False): '''Slice the report on property ``prop``.''' @@ -67,7 +71,15 @@ def slice(self, prop, when=None, unique=False): def case(self, check, part, env): c, p, e = check.name, part.fullname, env.name - return self._cases_index.get((c, p, e)) + ret = self._cases_index.get((c, p, e)) + if ret is None: + # Look up the case in the fallback reports + for rpt in self._fallbacks: + ret = rpt._cases_index.get((c, p, e)) + if ret is not None: + break + + return ret def restore_dangling(self, graph): '''Restore dangling dependencies in graph from the report data. @@ -89,7 +101,7 @@ def _do_restore(self, testcase): if tc is None: raise errors.ReframeError( f'could not restore testcase {testcase!r}: ' - f'not found in the report file' + f'not found in the report files' ) dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json') @@ -120,7 +132,7 @@ def next_report_filename(filepatt, new=True): return filepatt.format(sessionid=new_id) -def load_report(filename): +def _load_report(filename): try: with open(filename) as fp: report = json.load(fp) @@ -152,3 +164,14 @@ def load_report(filename): ) return _RunReport(report) + + +def load_report(*filenames): + primary = filenames[0] + rpt = _load_report(primary) + + # Add fallback reports + for f in filenames[1:]: + rpt.add_fallback(_load_report(f)) + + return rpt From 54eb660cbedba0f8c7816350f120a1f953dee7c5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 15 May 2021 01:04:22 +0200 Subject: [PATCH 4/6] Add unit test --- unittests/test_policies.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index f6b3fe3862..23742d2fd7 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -874,6 +874,28 @@ def test_restore_session(report_file, make_runner, assert new_report['runs'][0]['num_cases'] == 1 assert new_report['runs'][0]['testcases'][0]['name'] == 'T1' + # Generate an empty report and load it as primary with the original report + # as a fallback, in order to test if the dependencies are still resolved + # correctly + empty_report = tmp_path / 'empty.json' + + with open(empty_report, 'w') as fp: + empty_run = [ + { + 'num_cases': 0, + 'num_failures': 0, + 'num_aborted': 0, + 'num_skipped': 0, + 'runid': 0, + 'testcases': [] + } + ] + jsonext.dump(_generate_runreport(empty_run, *tm.timestamps()), fp) + + report2 = runreport.load_report(empty_report, report_file) + restored_cases = report2.restore_dangling(testgraph)[1] + assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'} + # Remove the test case dump file and retry os.remove(tmp_path / 'stage' / 'generic' / 'default' / 'builtin' / 'T4' / '.rfm_testcase.json') From db3297240670f1e8746bdedb657a7dc2b0c20e54 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 15 May 2021 01:13:52 +0200 Subject: [PATCH 5/6] Update documentation --- docs/manpage.rst | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index b759cb1ed0..77adf6d8a9 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -337,11 +337,11 @@ Options controlling ReFrame execution .. versionadded:: 3.2 -.. option:: --restore-session [REPORT] +.. option:: --restore-session [REPORT1[,REPORT2,...]] Restore a testing session that has run previously. - ``REPORT`` is a run report file generated by ReFrame. - If ``REPORT`` is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option). + ``REPORT1`` etc. are a run report files generated by ReFrame. + If a report is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option). If passed alone, this option will simply rerun all the test cases that have run previously based on the report file data. It is more useful to combine this option with any of the `test filtering <#test-filtering>`__ options, in which case only the selected test cases will be executed. The difference in test selection process when using this option is that the dependencies of the selected tests will not be selected for execution, as they would normally, but they will be restored. @@ -349,12 +349,18 @@ Options controlling ReFrame execution However, by doing ``reframe -n T1 --restore-session -r``, only ``T1`` would run and its immediate dependence ``T2`` will be restored. This is useful when you have deep test dependencies or some of the tests in the dependency chain are very time consuming. + Multiple reports may be passed as a comma-separated list. + ReFrame will try to restore any required test case by looking it up in each report sequentially. + If it cannot find it, it will issue an error and exit. + .. note:: In order for a test case to be restored, its stage directory must be present. This is not a problem when rerunning a failed case, since the stage directories of its dependencies are automatically kept, but if you want to rerun a successful test case, you should make sure to have run with the :option:`--keep-stage-files` option. .. versionadded:: 3.4 + .. versionchanged:: 3.6.1 + Multiple report files are now accepted. ---------------------------------- Options controlling job submission From ce62890f3c82590e5bd9988fe7900209aea100f5 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 15 May 2021 01:35:21 +0200 Subject: [PATCH 6/6] Fix PEP8 issues --- reframe/frontend/ci.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/ci.py b/reframe/frontend/ci.py index afa2d1c81a..c39d92e1a0 100644 --- a/reframe/frontend/ci.py +++ b/reframe/frontend/ci.py @@ -27,7 +27,9 @@ def rfm_command(testcase): report_file = f'{testcase.check.name}-report.json' if testcase.level: - restore_files = ','.join([f'{t.check.name}-report.json' for t in tc.deps]) + restore_files = ','.join( + f'{t.check.name}-report.json' for t in tc.deps + ) else: restore_files = None