[bugfix] Generate run reports per test with --ci-generate

docs/manpage.rst

@@ -358,24 +358,30 @@ Options controlling ReFrame execution
    .. versionadded:: 3.2
 
 
-.. option:: --restore-session [REPORT]
+.. option:: --restore-session [REPORT1[,REPORT2,...]]
 
    Restore a testing session that has run previously.
-   ``REPORT`` is a run report file generated by ReFrame.
-   If ``REPORT`` is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option).
+   ``REPORT1`` etc. are a run report files generated by ReFrame.
+   If a report is not given, ReFrame will pick the last report file found in the default location of report files (see the :option:`--report-file` option).
    If passed alone, this option will simply rerun all the test cases that have run previously based on the report file data.
    It is more useful to combine this option with any of the `test filtering <#test-filtering>`__ options, in which case only the selected test cases will be executed.
    The difference in test selection process when using this option is that the dependencies of the selected tests will not be selected for execution, as they would normally, but they will be restored.
    For example, if test ``T1`` depends on ``T2`` and ``T2`` depends on ``T3``, then running ``reframe -n T1 -r`` would cause both ``T2`` and ``T3`` to run.
    However, by doing ``reframe -n T1 --restore-session -r``, only ``T1`` would run and its immediate dependence ``T2`` will be restored.
    This is useful when you have deep test dependencies or some of the tests in the dependency chain are very time consuming.
 
+   Multiple reports may be passed as a comma-separated list.
+   ReFrame will try to restore any required test case by looking it up in each report sequentially.
+   If it cannot find it, it will issue an error and exit.
+
    .. note::
       In order for a test case to be restored, its stage directory must be present.
       This is not a problem when rerunning a failed case, since the stage directories of its dependencies are automatically kept, but if you want to rerun a successful test case, you should make sure to have run with the :option:`--keep-stage-files` option.
 
    .. versionadded:: 3.4
 
+   .. versionchanged:: 3.6.1
+      Multiple report files are now accepted.
 
 ----------------------------------
 Options controlling job submission

reframe/frontend/ci.py

@@ -25,19 +25,21 @@ def rfm_command(testcase):
         else:
             config_opt = ''
 
-        report_file = f'rfm-report-{testcase.level}.json'
+        report_file = f'{testcase.check.name}-report.json'
         if testcase.level:
-            restore_file = f'rfm-report-{testcase.level - 1}.json'
+            restore_files = ','.join(
+                f'{t.check.name}-report.json' for t in tc.deps
+            )
         else:
-            restore_file = None
+            restore_files = None
 
         return ' '.join([
             program,
             f'--prefix={prefix}', config_opt,
             f'{" ".join("-c " + c for c in checkpath)}',
             f'-R' if recurse else '',
             f'--report-file={report_file}',
-            f'--restore-session={restore_file}' if restore_file else '',
+            f'--restore-session={restore_files}' if restore_files else '',
             '-n', testcase.check.name, '-r'
         ])
 
@@ -54,7 +56,7 @@ def rfm_command(testcase):
             'stage': f'rfm-stage-{tc.level}',
             'script': [rfm_command(tc)],
             'artifacts': {
-                'paths': [f'rfm-report-{tc.level}.json']
+                'paths': [f'{tc.check.name}-report.json']
             },
             'needs': [t.check.name for t in tc.deps]
         }

reframe/frontend/cli.py

@@ -624,16 +624,16 @@ def main():
 
     # Setup the check loader
     if options.restore_session is not None:
-        # We need to load the failed checks only from a report
+        # We need to load the failed checks only from a list of reports
         if options.restore_session:
-            filename = options.restore_session
+            filenames = options.restore_session.split(',')
         else:
-            filename = runreport.next_report_filename(
+            filenames = [runreport.next_report_filename(
                 osext.expandvars(site_config.get('general/0/report_file')),
                 new=False
-            )
+            )]
 
-        report = runreport.load_report(filename)
+        report = runreport.load_report(*filenames)
         check_search_path = list(report.slice('filename', unique=True))
         check_search_recursive = False
 
@@ -817,6 +817,8 @@ def _case_failed(t):
             printer.debug(dependencies.format_deps(testgraph))
             if options.restore_session is not None:
                 testgraph, restored_cases = report.restore_dangling(testgraph)
+                print(dependencies.format_deps(testgraph))
+                print(restored_cases)
 
         testcases = dependencies.toposort(
             testgraph,

reframe/frontend/runreport.py

@@ -24,6 +24,7 @@ class _RunReport:
 
     def __init__(self, report):
         self._report = report
+        self._fallbacks = []    # fallback reports
 
         # Index all runs by test case; if a test case has run multiple times,
         # only the last time will be indexed
@@ -44,6 +45,9 @@ def __getitem__(self, key):
     def __getattr__(self, name):
         return getattr(self._report, name)
 
+    def add_fallback(self, report):
+        self._fallbacks.append(report)
+
     def slice(self, prop, when=None, unique=False):
         '''Slice the report on property ``prop``.'''
 
@@ -68,7 +72,15 @@ def slice(self, prop, when=None, unique=False):
 
     def case(self, check, part, env):
         c, p, e = check.name, part.fullname, env.name
-        return self._cases_index.get((c, p, e))
+        ret = self._cases_index.get((c, p, e))
+        if ret is None:
+            # Look up the case in the fallback reports
+            for rpt in self._fallbacks:
+                ret = rpt._cases_index.get((c, p, e))
+                if ret is not None:
+                    break
+
+        return ret
 
     def restore_dangling(self, graph):
         '''Restore dangling dependencies in graph from the report data.
@@ -90,7 +102,7 @@ def _do_restore(self, testcase):
         if tc is None:
             raise errors.ReframeError(
                 f'could not restore testcase {testcase!r}: '
-                f'not found in the report file'
+                f'not found in the report files'
             )
 
         dump_file = os.path.join(tc['stagedir'], '.rfm_testcase.json')
@@ -121,7 +133,7 @@ def next_report_filename(filepatt, new=True):
     return filepatt.format(sessionid=new_id)
 
 
-def load_report(filename):
+def _load_report(filename):
     try:
         with open(filename) as fp:
             report = json.load(fp)
@@ -155,6 +167,17 @@ def load_report(filename):
     return _RunReport(report)
 
 
+def load_report(*filenames):
+    primary = filenames[0]
+    rpt = _load_report(primary)
+
+    # Add fallback reports
+    for f in filenames[1:]:
+        rpt.add_fallback(_load_report(f))
+
+    return rpt
+
+
 def junit_xml_report(json_report):
     '''Generate a JUnit report from a standard ReFrame JSON report.'''
 

unittests/test_policies.py

@@ -853,6 +853,28 @@ def test_restore_session(report_file, make_runner,
     assert new_report['runs'][0]['num_cases'] == 1
     assert new_report['runs'][0]['testcases'][0]['name'] == 'T1'
 
+    # Generate an empty report and load it as primary with the original report
+    # as a fallback, in order to test if the dependencies are still resolved
+    # correctly
+    empty_report = tmp_path / 'empty.json'
+
+    with open(empty_report, 'w') as fp:
+        empty_run = [
+            {
+                'num_cases': 0,
+                'num_failures': 0,
+                'num_aborted': 0,
+                'num_skipped': 0,
+                'runid': 0,
+                'testcases': []
+            }
+        ]
+        jsonext.dump(_generate_runreport(empty_run, *tm.timestamps()), fp)
+
+    report2 = runreport.load_report(empty_report, report_file)
+    restored_cases = report2.restore_dangling(testgraph)[1]
+    assert {tc.check.name for tc in restored_cases} == {'T4', 'T5'}
+
     # Remove the test case dump file and retry
     os.remove(tmp_path / 'stage' / 'generic' / 'default' /
               'builtin' / 'T4' / '.rfm_testcase.json')