diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 6b34274f5e..9b46426d0f 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1111,6 +1111,16 @@ General Configuration Purge any loaded environment modules before running any tests. +.. js:attribute:: .general[].report_file + + :required: No + :default: ``"${HOME}/.reframe/reports/run-report-{sessionid}.json"`` + + The file where ReFrame will store its report. + + .. versionadded:: 3.1 + + .. js:attribute:: .general[].save_log_files :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index 520a6b9599..0722a51a2b 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -225,6 +225,8 @@ Options controlling ReFrame output Normally, if the stage directory of a test exists, ReFrame will remove it and recreate it. This option disables this behavior. + This option can also be set using the :envvar:`RFM_CLEAN_STAGEDIR` environment variable or the :js:attr:`clean_stagedir` general configuration parameter. + .. versionadded:: 3.1 .. option:: --save-log-files @@ -236,6 +238,16 @@ Options controlling ReFrame output This option can also be set using the :envvar:`RFM_SAVE_LOG_FILES` environment variable or the :js:attr:`save_log_files` general configuration parameter. +.. option:: --report-file=FILE + + The file where ReFrame will store its report. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. + + This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter. + + .. versionadded:: 3.1 + + ------------------------------------- Options controlling ReFrame execution ------------------------------------- @@ -790,6 +802,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_REPORT_FILE + + The file where ReFrame will store its report. + + .. versionadded:: 3.1 + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--report-file` + Associated configuration parameter :js:attr:`report_file` general configuration parameter + ================================== ================== + + .. envvar:: RFM_SAVE_LOG_FILES Save ReFrame log files in the output directory before exiting. diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index b57a740c3b..0371483666 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -90,28 +90,28 @@ Now it's time to run our first test: .. code-block:: none [ReFrame Setup] - version: 3.1-dev0 (rev: 986c3505) - command: './bin/reframe -c tutorials/basics/hello/hello1.py -r' - launched by: user@tresa.local - working directory: '/Users/user/reframe' + version: 3.1-dev2 (rev: 272e1aae) + command: ./bin/reframe -c tutorials/basics/hello/hello1.py -r + launched by: user@dhcp-133-44.cscs.ch + working directory: '/Users/user/Repositories/reframe' settings file: '' - check search path: '/Users/user/reframe/tutorials/basics/hello/hello1.py' - stage directory: '/Users/user/reframe/stage' - output directory: '/Users/user/reframe/output' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' [==========] Running 1 check(s) - [==========] Started on Sat Jun 20 09:44:52 2020 + [==========] Started on Fri Jul 24 11:05:46 2020 [----------] started processing HelloTest (HelloTest) [ RUN ] HelloTest on generic:default using builtin [----------] finished processing HelloTest (HelloTest) [----------] waiting for spawned checks to finish - [ OK ] (1/1) HelloTest on generic:default using builtin [compile: 0.735s run: 0.505s total: 1.272s] + [ OK ] (1/1) HelloTest on generic:default using builtin [compile: 0.378s run: 0.299s total: 0.712s] [----------] all spawned checks have finished [ PASSED ] Ran 1 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Sat Jun 20 09:44:53 2020 + [==========] Finished on Fri Jul 24 11:05:47 2020 Perfect! We have verified that we have a functioning C compiler in our system. @@ -121,7 +121,7 @@ On successful outcome of the test, the stage directory is removed by default, bu The prefixes of these directories are printed in the first section of the output. Let's inspect what files ReFrame produced for this test: -.. code-block:: bash +.. code-block:: console ls output/generic/default/builtin/HelloTest/ @@ -133,6 +133,75 @@ Let's inspect what files ReFrame produced for this test: ReFrame stores in the output directory of the test the build and run scripts it generated for building and running the code along with their standard output and error. All these files are prefixed with ``rfm_``. +ReFrame also generates a detailed JSON report for the whole regression testing session. +By default, this is stored inside the ``${HOME}/.reframe/reports`` directory and a new report file is generated every time ReFrame is run, but you can control this through the :option:`--report-file` command-line option. + +Here are the contents of the report file for our first ReFrame run: + + +.. code-block:: console + + cat ~/.reframe/reports/run-report-0.json + +.. code-block:: javascript + + { + "session_info": { + "cmdline": "./bin/reframe -c tutorials/basics/hello/hello1.py -r", + "config_file": "", + "data_version": "1.0", + "hostname": "dhcp-133-44.cscs.ch", + "prefix_output": "/Users/user/Repositories/reframe/output", + "prefix_stage": "/Users/user/Repositories/reframe/stage", + "user": "user", + "version": "3.1-dev2 (rev: 272e1aae)", + "workdir": "/Users/user/Repositories/reframe", + "time_start": "2020-07-24T11:05:46+0200", + "time_end": "2020-07-24T11:05:47+0200", + "time_elapsed": 0.7293069362640381, + "num_cases": 1, + "num_failures": 0 + }, + "runs": [ + { + "num_cases": 1, + "num_failures": 0, + "runid": 0, + "testcases": [ + { + "build_stderr": "rfm_HelloTest_build.err", + "build_stdout": "rfm_HelloTest_build.out", + "description": "HelloTest", + "environment": "builtin", + "fail_reason": null, + "fail_phase": null, + "jobid": 85063, + "job_stderr": "rfm_HelloTest_job.err", + "job_stdout": "rfm_HelloTest_job.out", + "name": "HelloTest", + "maintainers": [], + "nodelist": [ + "dhcp-133-44.cscs.ch" + ], + "outputdir": "/Users/user/Repositories/reframe/output/generic/default/builtin/HelloTest", + "perfvars": null, + "result": "success", + "stagedir": null, + "scheduler": "local", + "system": "generic:default", + "tags": [], + "time_compile": 0.3776402473449707, + "time_performance": 4.506111145019531e-05, + "time_run": 0.2992382049560547, + "time_sanity": 0.0005609989166259766, + "time_setup": 0.0031709671020507812, + "time_total": 0.7213571071624756 + } + ] + } + ] + } + More of "Hello, World!" ----------------------- diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a17205113c..b30ae0c860 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -925,6 +925,10 @@ def stderr(self): ''' return self._job.stderr + @property + def build_job(self): + return self._build_job + @property @sn.sanity_function def build_stdout(self): diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 2e67b6904b..6058232868 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -189,12 +189,12 @@ def json(self): 'container_platforms': [ { 'type': ctype, - 'modules': [m.name for m in cpenv.modules], + 'modules': [m for m in cpenv.modules], 'variables': [[n, v] for n, v in cpenv.variables.items()] } for ctype, cpenv in self._container_environs.items() ], - 'modules': [m.name for m in self._local_env.modules], + 'modules': [m for m in self._local_env.modules], 'variables': [[n, v] for n, v in self._local_env.variables.items()], 'environs': [e.name for e in self._environs], diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 97771a6a04..074568cd54 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -9,6 +9,7 @@ import re import socket import sys +import time import traceback import reframe @@ -69,6 +70,23 @@ def list_checks(checks, printer, detailed=False): printer.info('\nFound %d check(s).' % len(checks)) +def generate_report_filename(filepatt): + if '{sessionid}' not in filepatt: + return filepatt + + search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') + new_id = -1 + basedir = os.path.dirname(filepatt) or '.' + for filename in os.listdir(basedir): + match = re.match(search_patt, filename) + if match: + found_id = int(match.group(1)) + new_id = max(found_id, new_id) + + new_id += 1 + return filepatt.format(sessionid=new_id) + + def main(): # Setup command line options argparser = argparse.ArgumentParser() @@ -136,6 +154,12 @@ def main(): help='Save ReFrame log files to the output directory', envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) + output_options.add_argument( + '--report-file', action='store', metavar='FILE', + help="Store JSON run report in FILE", + envvar='RFM_REPORT_FILE', + configvar='general/report_file' + ) # Check discovery options locate_options.add_argument( @@ -509,19 +533,33 @@ def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") + session_info = { + 'cmdline': ' '.join(sys.argv), + 'config_file': rt.site_config.filename, + 'data_version': '1.0', + 'hostname': socket.gethostname(), + 'prefix_output': rt.output_prefix, + 'prefix_stage': rt.stage_prefix, + 'user': os_ext.osuser(), + 'version': os_ext.reframe_version(), + 'workdir': os.getcwd(), + } + # Print command line printer.info(f"[ReFrame Setup]") - print_infoline('version', os_ext.reframe_version()) - print_infoline('command', repr(' '.join(sys.argv))) - print_infoline('launched by', - f"{os_ext.osuser() or ''}@{socket.gethostname()}") - print_infoline('working directory', repr(os.getcwd())) - print_infoline('settings file', f'{site_config.filename!r}') + print_infoline('version', session_info['version']) + print_infoline('command', repr(session_info['cmdline'])) + print_infoline( + f"launched by", + f"{session_info['user'] or ''}@{session_info['hostname']}" + ) + print_infoline('working directory', repr(session_info['workdir'])) + print_infoline('settings file', f"{session_info['config_file']!r}") print_infoline('check search path', f"{'(R) ' if loader.recurse else ''}" f"{':'.join(loader.load_path)!r}") - print_infoline('stage directory', repr(rt.stage_prefix)) - print_infoline('output directory', repr(rt.output_prefix)) + print_infoline('stage directory', repr(session_info['prefix_stage'])) + print_infoline('output directory', repr(session_info['prefix_output'])) printer.info('') try: # Locate and load checks @@ -696,8 +734,18 @@ def print_infoline(param, value): max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: + time_start = time.time() + session_info['time_start'] = time.strftime( + '%FT%T%z', time.localtime(time_start), + ) runner.runall(testcases) finally: + time_end = time.time() + session_info['time_end'] = time.strftime( + '%FT%T%z', time.localtime(time_end) + ) + session_info['time_elapsed'] = time_end - time_start + # Print a retry report if we did any retries if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) @@ -712,6 +760,33 @@ def print_infoline(param, value): if options.performance_report: printer.info(runner.stats.performance_report()) + # Generate the report for this session + report_file = os.path.normpath( + os_ext.expandvars(rt.get_option('general/0/report_file')) + ) + basedir = os.path.dirname(report_file) + if basedir: + os.makedirs(basedir, exist_ok=True) + + # Build final JSON report + run_stats = runner.stats.json() + session_info.update({ + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'] + }) + json_report = { + 'session_info': session_info, + 'runs': run_stats + } + report_file = generate_report_filename(report_file) + try: + with open(report_file, 'w') as fp: + json.dump(json_report, fp, indent=2) + except OSError as e: + printer.warning( + f'failed to generate report in {report_file!r}: {e}' + ) + else: printer.error("No action specified. Please specify `-l'/`-L' for " "listing or `-r' for running. " diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index b2760041fe..ae7ece0f43 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD-3-Clause import reframe.core.runtime as rt -from reframe.core.exceptions import StatisticsError +from reframe.core.exceptions import format_exception, StatisticsError class TestStats: @@ -12,18 +12,21 @@ class TestStats: def __init__(self): # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] - self._tasks = [[]] + self._alltasks = [[]] + + # Data collected for all the runs of this session in JSON format + self._run_data = [] def add_task(self, task): current_run = rt.runtime().current_run - if current_run == len(self._tasks): - self._tasks.append([]) + if current_run == len(self._alltasks): + self._alltasks.append([]) - self._tasks[current_run].append(task) + self._alltasks[current_run].append(task) def tasks(self, run=-1): try: - return self._tasks[run] + return self._alltasks[run] except IndexError: raise StatisticsError('no such run: %s' % run) from None @@ -43,8 +46,7 @@ def retry_report(self): report.append('SUMMARY OF RETRIES') report.append(line_width * '-') messages = {} - - for run in range(1, len(self._tasks)): + for run in range(1, len(self._alltasks)): for t in self.tasks(run): partition_name = '' environ_name = '' @@ -66,46 +68,131 @@ def retry_report(self): return '\n'.join(report) + def json(self, force=False): + if not force and self._run_data: + return self._run_data + + for runid, run in enumerate(self._alltasks): + testcases = [] + num_failures = 0 + for t in run: + check = t.check + partition = check.current_partition + entry = { + 'build_stderr': None, + 'build_stdout': None, + 'description': check.descr, + 'environment': None, + 'fail_reason': None, + 'fail_phase': None, + 'jobid': None, + 'job_stderr': None, + 'job_stdout': None, + 'name': check.name, + 'maintainers': check.maintainers, + 'nodelist': [], + 'outputdir': None, + 'perfvars': None, + 'result': None, + 'stagedir': None, + 'scheduler': None, + 'system': check.current_system.name, + 'tags': list(check.tags), + 'time_compile': t.duration('compile_complete'), + 'time_performance': t.duration('performance'), + 'time_run': t.duration('run_complete'), + 'time_sanity': t.duration('sanity'), + 'time_setup': t.duration('setup'), + 'time_total': t.duration('total') + } + partition = check.current_partition + environ = check.current_environ + if partition: + entry['system'] = partition.fullname + entry['scheduler'] = partition.scheduler.registered_name + + if environ: + entry['environment'] = environ.name + + if check.job: + entry['jobid'] = check.job.jobid + entry['job_stderr'] = check.stderr.evaluate() + entry['job_stdout'] = check.stdout.evaluate() + entry['nodelist'] = check.job.nodelist or [] + + if check.build_job: + entry['build_stderr'] = check.build_stderr.evaluate() + entry['build_stdout'] = check.build_stdout.evaluate() + + if t.failed: + num_failures += 1 + entry['result'] = 'failure' + entry['stagedir'] = check.stagedir + entry['fail_phase'] = t.failed_stage + if t.exc_info is not None: + entry['fail_reason'] = format_exception(*t.exc_info) + else: + entry['result'] = 'success' + entry['outputdir'] = check.outputdir + + if check.perf_patterns: + # Record performance variables + entry['perfvars'] = [] + for key, ref in check.perfvalues.items(): + var = key.split(':')[-1] + val, ref, lower, upper, unit = ref + entry['perfvars'].append({ + 'name': var, + 'reference': ref, + 'thres_lower': lower, + 'thres_upper': upper, + 'unit': unit, + 'value': val + }) + + testcases.append(entry) + + self._run_data.append({ + 'num_cases': len(run), + 'num_failures': num_failures, + 'runid': runid, + 'testcases': testcases + }) + + return self._run_data + def failure_report(self): line_width = 78 report = [line_width * '='] report.append('SUMMARY OF FAILURES') - current_run = rt.runtime().current_run - for tf in (t for t in self.tasks(current_run) if t.failed): - check = tf.check - partition = check.current_partition - partname = partition.fullname if partition else 'None' - environ_name = (check.current_environ.name - if check.current_environ else 'None') - retry_info = ('(for the last of %s retries)' % current_run - if current_run > 0 else '') - + run_report = self.json()[-1] + last_run = run_report['runid'] + for r in run_report['testcases']: + if r['result'] == 'success': + continue + + retry_info = ( + f'(for the last of {last_run} retries)' if last_run > 0 else '' + ) report.append(line_width * '-') - report.append('FAILURE INFO for %s %s' % (check.name, retry_info)) - report.append(' * Test Description: %s' % check.descr) - report.append(' * System partition: %s' % partname) - report.append(' * Environment: %s' % environ_name) - report.append(' * Stage directory: %s' % check.stagedir) - report.append(' * Node list: %s' % - (','.join(check.job.nodelist) - if check.job and check.job.nodelist else '')) - job_type = 'local' if check.is_local() else 'batch job' - jobid = check.job.jobid if check.job else -1 - report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) - report.append(' * Maintainers: %s' % check.maintainers) - report.append(' * Failing phase: %s' % tf.failed_stage) - report.append(" * Rerun with '-n %s -p %s --system %s'" % - (check.name, environ_name, partname)) - reason = ' * Reason: ' - if tf.exc_info is not None: - from reframe.core.exceptions import format_exception - - reason += format_exception(*tf.exc_info) - report.append(reason) - - elif tf.failed_stage == 'check_sanity': + report.append(f"FAILURE INFO for {r['name']} {retry_info}") + report.append(f" * Test Description: {r['description']}") + report.append(f" * System partition: {r['system']}") + report.append(f" * Environment: {r['environment']}") + report.append(f" * Stage directory: {r['stagedir']}") + nodelist = ','.join(r['nodelist']) if r['nodelist'] else None + report.append(f" * Node list: {nodelist}") + job_type = 'local' if r['scheduler'] == 'local' else 'batch job' + jobid = r['jobid'] + report.append(f" * Job type: {job_type} (id={r['jobid']})") + report.append(f" * Maintainers: {r['maintainers']}") + report.append(f" * Failing phase: {r['fail_phase']}") + report.append(f" * Rerun with '-n {r['name']}" + f" -p {r['environment']} --system {r['system']}'") + report.append(f" * Reason: {r['fail_reason']}") + if r['fail_phase'] == 'sanity': report.append('Sanity check failure') - elif tf.failed_stage == 'check_performance': + elif r['fail_phase'] == 'performance': report.append('Performance check failure') else: # This shouldn't happen... @@ -120,10 +207,10 @@ def failure_stats(self): for tf in (t for t in self.tasks(current_run) if t.failed): check = tf.check partition = check.current_partition - partname = partition.fullname if partition else 'None' + partfullname = partition.fullname if partition else 'None' environ_name = (check.current_environ.name if check.current_environ else 'None') - f = f'[{check.name}, {environ_name}, {partname}]' + f = f'[{check.name}, {environ_name}, {partfullname}]' if tf.failed_stage not in failures: failures[tf.failed_stage] = [] @@ -159,6 +246,8 @@ def failure_stats(self): return '' def performance_report(self): + # FIXME: Adapt this function to use the JSON report + line_width = 78 report_start = line_width * '=' report_title = 'PERFORMANCE REPORT' diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index e9230f082e..b2724e4891 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -163,7 +163,10 @@ "descr": {"type": "string"}, "scheduler": { "type": "string", - "enum": ["local", "pbs", "slurm", "squeue", "torque"] + "enum": [ + "local", "pbs", "slurm", + "squeue", "torque" + ] }, "launcher": { "type": "string", @@ -361,6 +364,7 @@ }, "non_default_craype": {"type": "boolean"}, "purge_environment": {"type": "boolean"}, + "report_file": {"type": "string"}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, @@ -403,6 +407,7 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/purge_environment": false, + "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json new file mode 100644 index 0000000000..b26f15ec56 --- /dev/null +++ b/reframe/schemas/runreport.json @@ -0,0 +1,115 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/eth-cscs/reframe/master/schemas/runreport.json", + "title": "Validation schema for ReFrame's run report", + "type": "object", + "properties": { + "session_info": { + "type": "object", + "properties": { + "cmdline": {"type": "string"}, + "config_file": {"type": ["string", "null"]}, + "data_version": {"type": "string"}, + "hostname": {"type": "string"}, + "num_cases": {"type": "number"}, + "num_failures": {"type": "number"}, + "prefix_output": {"type": "string"}, + "prefix_stage": {"type": "string"}, + "time_elapsed": {"type": "number"}, + "time_end": {"type": "string"}, + "time_start": {"type": "string"}, + "user": {"type": "string"}, + "version": {"type": "string"}, + "workdir": {"type": "string"} + }, + "required": ["data_version"] + }, + "runs": { + "type": "array", + "items": { + "type": "object", + "properties": { + "num_cases": {"type": "number"}, + "num_failures": {"type": "number"}, + "runid": {"type": "number"}, + "testcases": { + "type": "array", + "items": { + "type": "object", + "properties": { + "build_stderr": {"type": ["string", "null"]}, + "build_stdout": {"type": ["string", "null"]}, + "description": {"type": "string"}, + "environment": {"type": ["string", "null"]}, + "fail_phase": {"type": ["string", "null"]}, + "fail_reason": {"type": ["string", "null"]}, + "jobid": {"type": ["number", "null"]}, + "job_stderr": {"type": ["string", "null"]}, + "job_stdout": {"type": ["string", "null"]}, + "name": {"type": "string"}, + "maintainers": { + "type": "array", + "items": {"type": "string"} + }, + "nodelist": { + "type": "array", + "items": {"type": "string"} + }, + "outputdir": {"type": ["string", "null"]}, + "perfvars": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "reference": { + "type": ["number", "null"] + }, + "thres_lower": { + "type": ["number", "null"] + }, + "thres_upper": { + "type": ["number", "null"] + }, + "unit": {"type": ["string", "null"]}, + "value": {"type": "number"} + }, + "required": [ + "name", "reference", + "thres_lower", "thres_upper", + "unit", "value" + ] + } + }, + "result": { + "type": "string", + "enum": ["success", "failure"] + }, + "scheduler": {"type": ["string", "null"]}, + "stagedir": {"type": ["string", "null"]}, + "system": {"type": "string"}, + "tags": { + "type": "array", + "items": {"type": "string"} + }, + "time_compile": {"type": ["number", "null"]}, + "time_performance": {"type": ["number", "null"]}, + "time_run": {"type": ["number", "null"]}, + "time_sanity": {"type": ["number", "null"]}, + "time_setup": {"type": ["number", "null"]}, + "time_total": {"type": ["number", "null"]} + }, + "required": [ + "environment", "name", "result", "system" + ] + }, + "required": [ + "num_cases", "num_failures", "runid", "testcases" + ] + } + } + } + } + }, + "required": ["runs"] +} diff --git a/unittests/test_cli.py b/unittests/test_cli.py index a5cf59b1ea..27e5e10fb4 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -75,7 +75,10 @@ def _run_reframe(system='generic:default', perflogdir=str(perflogdir)): import reframe.frontend.cli as cli - argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor'] + # We always pass the --report-file option, because we don't want to + # pollute the user's home directory + argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor', + f'--report-file={tmp_path / "report.json"}'] if mode: argv += ['--mode', mode] @@ -145,7 +148,19 @@ def test_check_success(run_reframe, tmp_path, logfile): assert 'PASSED' in stdout assert 'FAILED' not in stdout assert returncode == 0 - os.path.exists(tmp_path / 'output' / logfile) + assert os.path.exists(tmp_path / 'output' / logfile) + assert os.path.exists(tmp_path / 'report.json') + + +def test_report_file_with_sessionid(run_reframe, tmp_path): + returncode, stdout, _ = run_reframe( + more_options=[ + f'--save-log-files', + f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}' + ] + ) + assert returncode == 0 + assert os.path.exists(tmp_path / 'rfm-report-0.json') def test_check_submit_success(run_reframe, remote_exec_ctx): diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 2ea2309870..1df2975fa8 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,9 +3,15 @@ # # SPDX-License-Identifier: BSD-3-Clause +import json +import jsonschema import os import pytest +import socket +import sys +import time +import reframe import reframe.core.runtime as rt import reframe.frontend.dependency as dependency import reframe.frontend.executors as executors @@ -105,18 +111,54 @@ def num_failures_stage(runner, stage): return len([t for t in stats.failures() if t.failed_stage == stage]) +def _validate_runreport(report): + schema_filename = 'reframe/schemas/runreport.json' + with open(schema_filename) as fp: + schema = json.loads(fp.read()) + + jsonschema.validate(report, schema) + + def test_runall(make_runner, make_cases, common_exec_ctx): runner = make_runner() + time_start = time.time() runner.runall(make_cases()) - stats = runner.stats - assert 8 == stats.num_cases() + time_end = time.time() + assert 8 == runner.stats.num_cases() assert_runall(runner) - assert 5 == len(stats.failures()) + assert 5 == len(runner.stats.failures()) assert 2 == num_failures_stage(runner, 'setup') assert 1 == num_failures_stage(runner, 'sanity') assert 1 == num_failures_stage(runner, 'performance') assert 1 == num_failures_stage(runner, 'cleanup') + # Create a run report and validate it + run_stats = runner.stats.json() + report = { + 'session_info': { + 'cmdline': ' '.join(sys.argv), + 'config_file': rt.runtime().site_config.filename, + 'data_version': '1.0', + 'hostname': socket.gethostname(), + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'], + 'prefix_output': rt.runtime().output_prefix, + 'prefix_stage': rt.runtime().stage_prefix, + 'time_elapsed': time_end - time_start, + 'time_end': time.strftime( + '%FT%T%z', time.localtime(time_end), + ), + 'time_start': time.strftime( + '%FT%T%z', time.localtime(time_start), + ), + 'user': os_ext.osuser(), + 'version': os_ext.reframe_version(), + 'workdir': os.getcwd() + }, + 'runs': run_stats + } + _validate_runreport(report) + def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): runner = make_runner()