From c6b0aa08e858ce2e0447e949f506d367f01526cc Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 18 Jun 2020 09:45:52 +0200 Subject: [PATCH 01/15] add json output report --- reframe/frontend/cli.py | 2 + reframe/frontend/statistics.py | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index f84615c93f..3201bc0284 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -702,6 +702,8 @@ def print_infoline(param, value): if options.performance_report: printer.info(runner.stats.performance_report()) + runner.stats.json_report() + else: printer.error("No action specified. Please specify `-l'/`-L' for " "listing or `-r' for running. " diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 37110de4d5..001f729f09 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -3,6 +3,8 @@ # # SPDX-License-Identifier: BSD-3-Clause +import json +import jsonschema import reframe.core.debug as debug import reframe.core.runtime as rt from reframe.core.exceptions import StatisticsError @@ -70,6 +72,75 @@ def retry_report(self): return '\n'.join(report) + def output_dict(self): + report = [] + current_run = rt.runtime().current_run + for t in self.tasks(current_run): + check = t.check + partition = check.current_partition + partname = partition.fullname if partition else 'None' + environ_name = (check.current_environ.name + if check.current_environ else 'None') + nodelist = (','.join(check.job.nodelist) + if check.job and check.job.nodelist else '') + job_type = 'local' if check.is_local() else 'batch job' + jobid = check.job.jobid if check.job else -1 + if t.failed: + result = 'fail' + else: + result = 'success' + + report.append({ + 'test': check.name, + 'description': check.descr, + 'result': result, + 'system': partname, + 'environment': environ_name, + 'stagedir': check.stagedir, + 'outputdir': check.outputdir, + 'nodelist': nodelist, + 'jobtype': job_type, + 'jobid': jobid, + 'maintainers': check.maintainers, + 'tags': list(check.tags), + 'retries': current_run + }) + + return report + + def json_report(self): + schema = { + "type": "array", + "items": { + "type": "object", + "properties": { + "test": {"type": "string"}, + "description": {"type": "string"}, + "result": {"type": "string"}, + "system": {"type": "string"}, + "environment": {"type": "string"}, + "stagedir": {"type": "string"}, + "outputdir": {"type": "string"}, + "nodelist": {"type": "string"}, + "jobtype": {"type": "string"}, + "jobid": {"type": "number"}, + "maintainers": { + "type": "array", + "items": {"type": "string"} + }, + "tags": { + "type": "array", + "items": {"type": "string"} + }, + "retries": {"type": "number"} + } + }, + } + report = self.output_dict() + jsonschema.validate(instance=report, schema=schema) + with open('report.json', 'w') as fp: + json.dump(report, fp, indent=4) + def failure_report(self): line_width = 78 report = [line_width * '='] From 6b9518df06440356e0af01b6049032ea2e872c03 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 23 Jun 2020 09:31:03 +0200 Subject: [PATCH 02/15] refactor the dict construction --- reframe/frontend/statistics.py | 69 +++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 001f729f09..899f2c5f33 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -81,30 +81,46 @@ def output_dict(self): partname = partition.fullname if partition else 'None' environ_name = (check.current_environ.name if check.current_environ else 'None') - nodelist = (','.join(check.job.nodelist) - if check.job and check.job.nodelist else '') job_type = 'local' if check.is_local() else 'batch job' jobid = check.job.jobid if check.job else -1 - if t.failed: - result = 'fail' - else: - result = 'success' - - report.append({ - 'test': check.name, + report_dict = { + 'name': check.name, 'description': check.descr, - 'result': result, 'system': partname, 'environment': environ_name, - 'stagedir': check.stagedir, - 'outputdir': check.outputdir, - 'nodelist': nodelist, - 'jobtype': job_type, - 'jobid': jobid, - 'maintainers': check.maintainers, 'tags': list(check.tags), - 'retries': current_run - }) + 'maintainers': check.maintainers, + 'scheduler': check.job.scheduler.registered_name, + 'job_stdout': check.job.stdout, + 'job_stderr': check.job.stderr, + } + if not check.is_local(): + report_dict['jobid'] = check.job.jobid if check.job else -1 + report_dict['nodelist'] = (check.job.nodelist + if check.job and check.job.nodelist + else '') + + if check._build_job: + report_dict['build_stdout'] = check._build_job.stdout + report_dict['build_stderr'] = check._build_job.stderr + + if t.failed: + report_dict['result'] = 'fail' + if t.exc_info is not None: + from reframe.core.exceptions import format_exception + + report_dict['failing_reason'] = format_exception( + *t.exc_info) + report_dict['failing_phase'] = t.failed_stage + report_dict['stagedir'] = check.stagedir + else: + report_dict['result'] = 'success' + report_dict['outputdir'] = check.outputdir + + if current_run > 0: + report_dict['retries'] = current_run + + report.append(report_dict) return report @@ -114,16 +130,25 @@ def json_report(self): "items": { "type": "object", "properties": { - "test": {"type": "string"}, + "name": {"type": "string"}, "description": {"type": "string"}, - "result": {"type": "string"}, "system": {"type": "string"}, "environment": {"type": "string"}, "stagedir": {"type": "string"}, "outputdir": {"type": "string"}, - "nodelist": {"type": "string"}, - "jobtype": {"type": "string"}, + "nodelist": { + "type": "array", + "items": {"type": "string"} + }, + "scheduler": {"type": "string"}, "jobid": {"type": "number"}, + "result": {"type": "string"}, + "failing_phase": {"type": "string"}, + "failing_reason": {"type": "string"}, + "build_stdout": {"type": "string"}, + "build_stderr": {"type": "string"}, + "job_stdout": {"type": "string"}, + "job_stderr": {"type": "string"}, "maintainers": { "type": "array", "items": {"type": "string"} From 2a032a22e205a0d0207ee9c6634f28d08679a7d5 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 23 Jun 2020 14:14:13 +0200 Subject: [PATCH 03/15] write failure report from output dict --- reframe/frontend/statistics.py | 69 ++++++++++++++++------------------ 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 899f2c5f33..9660293d08 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -81,8 +81,6 @@ def output_dict(self): partname = partition.fullname if partition else 'None' environ_name = (check.current_environ.name if check.current_environ else 'None') - job_type = 'local' if check.is_local() else 'batch job' - jobid = check.job.jobid if check.job else -1 report_dict = { 'name': check.name, 'description': check.descr, @@ -90,15 +88,21 @@ def output_dict(self): 'environment': environ_name, 'tags': list(check.tags), 'maintainers': check.maintainers, - 'scheduler': check.job.scheduler.registered_name, - 'job_stdout': check.job.stdout, - 'job_stderr': check.job.stderr, + 'scheduler': 'None', + 'jobid': -1, + 'nodelist': [], + 'job_stdout': 'None', + 'job_stderr': 'None' } - if not check.is_local(): - report_dict['jobid'] = check.job.jobid if check.job else -1 + if check.job: + report_dict['scheduler'] = check.job.scheduler.registered_name + report_dict['jobid'] = (check.job.jobid if check.job.jobid + else -1) report_dict['nodelist'] = (check.job.nodelist - if check.job and check.job.nodelist - else '') + if check.job.nodelist + else []) + report_dict['job_stdout'] = check.job.stdout + report_dict['job_stderr'] = check.job.stderr if check._build_job: report_dict['build_stdout'] = check._build_job.stdout @@ -112,7 +116,8 @@ def output_dict(self): report_dict['failing_reason'] = format_exception( *t.exc_info) report_dict['failing_phase'] = t.failed_stage - report_dict['stagedir'] = check.stagedir + report_dict['stagedir'] = (check.stagedir if check.stagedir + else 'None') else: report_dict['result'] = 'success' report_dict['outputdir'] = check.outputdir @@ -171,41 +176,31 @@ def failure_report(self): report = [line_width * '='] report.append('SUMMARY OF FAILURES') current_run = rt.runtime().current_run - for tf in (t for t in self.tasks(current_run) if t.failed): - check = tf.check - partition = check.current_partition - partname = partition.fullname if partition else 'None' - environ_name = (check.current_environ.name - if check.current_environ else 'None') + for tf in (t for t in self.output_dict() if t['result'] == 'fail'): retry_info = ('(for the last of %s retries)' % current_run - if current_run > 0 else '') + if 'retries' in tf.keys() else '') report.append(line_width * '-') - report.append('FAILURE INFO for %s %s' % (check.name, retry_info)) - report.append(' * Test Description: %s' % check.descr) - report.append(' * System partition: %s' % partname) - report.append(' * Environment: %s' % environ_name) - report.append(' * Stage directory: %s' % check.stagedir) + report.append('FAILURE INFO for %s %s' % (tf['name'], retry_info)) + report.append(' * Test Description: %s' % tf['description']) + report.append(' * System partition: %s' % tf['system']) + report.append(' * Environment: %s' % tf['environment']) + report.append(' * Stage directory: %s' % tf['stagedir']) report.append(' * Node list: %s' % - (','.join(check.job.nodelist) - if check.job and check.job.nodelist else '')) - job_type = 'local' if check.is_local() else 'batch job' - jobid = check.job.jobid if check.job else -1 + (','.join(tf['nodelist']) + if tf['nodelist'] else 'None')) + job_type = 'local' if tf['scheduler'] == 'local' else 'batch job' + jobid = tf['jobid'] if tf['jobid'] > 0 else 'None' report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) - report.append(' * Maintainers: %s' % check.maintainers) - report.append(' * Failing phase: %s' % tf.failed_stage) + report.append(' * Maintainers: %s' % tf['maintainers']) + report.append(' * Failing phase: %s' % tf['failing_phase']) report.append(" * Rerun with '-n %s -p %s --system %s'" % - (check.name, environ_name, partname)) - reason = ' * Reason: ' - if tf.exc_info is not None: - from reframe.core.exceptions import format_exception - - reason += format_exception(*tf.exc_info) - report.append(reason) + (tf['name'], tf['environment'], tf['system'])) + report.append(" * Reason: %s" % tf['failing_reason']) - elif tf.failed_stage == 'check_sanity': + if tf['failing_phase'] == 'sanity': report.append('Sanity check failure') - elif tf.failed_stage == 'check_performance': + elif tf['failing_phase'] == 'performance': report.append('Performance check failure') else: # This shouldn't happen... From eb768b53cdf0c831c7454bdb260aea33e8598b46 Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 24 Jun 2020 16:46:55 +0200 Subject: [PATCH 04/15] refactor and add unittest --- reframe/frontend/statistics.py | 168 ++++++++++++++------------------- reframe/schemas/report.json | 41 ++++++++ unittests/test_policies.py | 21 +++++ 3 files changed, 131 insertions(+), 99 deletions(-) create mode 100644 reframe/schemas/report.json diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 9660293d08..71a597aa2b 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -4,10 +4,12 @@ # SPDX-License-Identifier: BSD-3-Clause import json -import jsonschema +import os + +import reframe import reframe.core.debug as debug import reframe.core.runtime as rt -from reframe.core.exceptions import StatisticsError +from reframe.core.exceptions import StatisticsError, ConfigError class TestStats: @@ -72,111 +74,79 @@ def retry_report(self): return '\n'.join(report) - def output_dict(self): - report = [] + @property + def report_dict(self): + _report = [] current_run = rt.runtime().current_run - for t in self.tasks(current_run): - check = t.check - partition = check.current_partition - partname = partition.fullname if partition else 'None' - environ_name = (check.current_environ.name - if check.current_environ else 'None') - report_dict = { - 'name': check.name, - 'description': check.descr, - 'system': partname, - 'environment': environ_name, - 'tags': list(check.tags), - 'maintainers': check.maintainers, - 'scheduler': 'None', - 'jobid': -1, - 'nodelist': [], - 'job_stdout': 'None', - 'job_stderr': 'None' - } - if check.job: - report_dict['scheduler'] = check.job.scheduler.registered_name - report_dict['jobid'] = (check.job.jobid if check.job.jobid - else -1) - report_dict['nodelist'] = (check.job.nodelist - if check.job.nodelist - else []) - report_dict['job_stdout'] = check.job.stdout - report_dict['job_stderr'] = check.job.stderr - - if check._build_job: - report_dict['build_stdout'] = check._build_job.stdout - report_dict['build_stderr'] = check._build_job.stderr - - if t.failed: - report_dict['result'] = 'fail' - if t.exc_info is not None: - from reframe.core.exceptions import format_exception - - report_dict['failing_reason'] = format_exception( - *t.exc_info) - report_dict['failing_phase'] = t.failed_stage - report_dict['stagedir'] = (check.stagedir if check.stagedir - else 'None') - else: - report_dict['result'] = 'success' - report_dict['outputdir'] = check.outputdir - - if current_run > 0: - report_dict['retries'] = current_run - - report.append(report_dict) - - return report + for tl, tr in zip(self._tasks, range(current_run + 1)): + for t in tl: + check = t.check + partition = check.current_partition + partname = partition.fullname if partition else None + environ_name = (check.current_environ.name + if check.current_environ else None) + report = { + 'name': check.name, + 'description': check.descr, + 'system': partname, + 'environment': environ_name, + 'tags': list(check.tags), + 'maintainers': check.maintainers, + 'scheduler': None, + 'jobid': None, + 'nodelist': [], + 'job_stdout': None, + 'job_stderr': None, + 'build_stdout': None, + 'build_stderr': None, + 'failing_reason': None, + 'failing_phase': None, + 'outputdir': None, + 'stagedir': None + } + if check.job: + report['scheduler'] = check.job.scheduler.registered_name + report['jobid'] = check.job.jobid + report['nodelist'] = (check.job.nodelist + if check.job.nodelist + else []) + report['job_stdout'] = check.job.stdout + report['job_stderr'] = check.job.stderr + + if check._build_job: + report['build_stdout'] = check._build_job.stdout + report['build_stderr'] = check._build_job.stderr + + if t.failed: + report['result'] = 'fail' + if t.exc_info is not None: + from reframe.core.exceptions import format_exception + + report['failing_reason'] = format_exception( + *t.exc_info) + report['failing_phase'] = t.failed_stage + report['stagedir'] = check.stagedir + else: + report['result'] = 'success' + report['outputdir'] = check.outputdir + + report['try'] = tr + + _report.append(report) + + return _report def json_report(self): - schema = { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "description": {"type": "string"}, - "system": {"type": "string"}, - "environment": {"type": "string"}, - "stagedir": {"type": "string"}, - "outputdir": {"type": "string"}, - "nodelist": { - "type": "array", - "items": {"type": "string"} - }, - "scheduler": {"type": "string"}, - "jobid": {"type": "number"}, - "result": {"type": "string"}, - "failing_phase": {"type": "string"}, - "failing_reason": {"type": "string"}, - "build_stdout": {"type": "string"}, - "build_stderr": {"type": "string"}, - "job_stdout": {"type": "string"}, - "job_stderr": {"type": "string"}, - "maintainers": { - "type": "array", - "items": {"type": "string"} - }, - "tags": { - "type": "array", - "items": {"type": "string"} - }, - "retries": {"type": "number"} - } - }, - } - report = self.output_dict() - jsonschema.validate(instance=report, schema=schema) with open('report.json', 'w') as fp: - json.dump(report, fp, indent=4) + json.dump(self.report_dict, fp, indent=4) def failure_report(self): line_width = 78 report = [line_width * '='] report.append('SUMMARY OF FAILURES') current_run = rt.runtime().current_run - for tf in (t for t in self.output_dict() if t['result'] == 'fail'): + for tf in (t for t in self.report_dict + if t['result'] == 'fail' and t['try'] == current_run): retry_info = ('(for the last of %s retries)' % current_run if 'retries' in tf.keys() else '') @@ -188,9 +158,9 @@ def failure_report(self): report.append(' * Stage directory: %s' % tf['stagedir']) report.append(' * Node list: %s' % (','.join(tf['nodelist']) - if tf['nodelist'] else 'None')) + if tf['nodelist'] else None)) job_type = 'local' if tf['scheduler'] == 'local' else 'batch job' - jobid = tf['jobid'] if tf['jobid'] > 0 else 'None' + jobid = tf['jobid'] report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) report.append(' * Maintainers: %s' % tf['maintainers']) report.append(' * Failing phase: %s' % tf['failing_phase']) diff --git a/reframe/schemas/report.json b/reframe/schemas/report.json new file mode 100644 index 0000000000..a9be4ce9f9 --- /dev/null +++ b/reframe/schemas/report.json @@ -0,0 +1,41 @@ +{ + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "description": {"type": ["string", "null"]}, + "system": {"type": ["string", "null"]}, + "environment": {"type": ["string", "null"]}, + "stagedir": {"type": ["string", "null"]}, + "outputdir": {"type": ["string", "null"]}, + "nodelist": { + "type": "array", + "items": {"type": ["string"]} + }, + "scheduler": {"type": ["string", "null"]}, + "jobid": {"type": ["number", "null"]}, + "result": {"type": "string"}, + "failing_phase": {"type": ["string", "null"]}, + "failing_reason": {"type": ["string", "null"]}, + "build_stdout": {"type": ["string", "null"]}, + "build_stderr": {"type": ["string", "null"]}, + "job_stdout": {"type": ["string", "null"]}, + "job_stderr": {"type": ["string", "null"]}, + "maintainers": { + "type": "array", + "items": {"type": ["string", "null"]} + }, + "tags": { + "type": "array", + "items": {"type": ["string", "null"]} + }, + "try": {"type": "number"} + }, + "additionalProperties": false, + "required": ["name", "description", "system", "environment", "stagedir", + "outputdir", "nodelist", "scheduler", "jobid", "result", + "failing_phase", "failing_reason", "build_stdout", "build_stderr", + "job_stdout", "job_stderr", "maintainers", "tags", "try"] + } +} diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 770ca21016..0f04f52162 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,9 +3,12 @@ # # SPDX-License-Identifier: BSD-3-Clause +import json +import jsonschema import os import pytest +import reframe import reframe.core.runtime as rt import reframe.frontend.dependency as dependency import reframe.frontend.executors as executors @@ -104,10 +107,28 @@ def num_failures_stage(runner, stage): return len([t for t in stats.failures() if t.failed_stage == stage]) +def open_json_schema(): + # Open and store the JSON schema for later validation + schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe', + 'schemas', 'report.json') + with open(schema_filename) as fp: + try: + schema = json.loads(fp.read()) + except json.JSONDecodeError as e: + raise ReframeFatalError( + f"invalid configuration schema: '{schema_filename}'" + ) from e + + return schema + + def test_runall(make_runner, make_cases, common_exec_ctx): runner = make_runner() runner.runall(make_cases()) stats = runner.stats + json_out = runner.stats.report_dict + schema = open_json_schema() + jsonschema.validate(json_out, schema) assert 8 == stats.num_cases() assert_runall(runner) assert 5 == len(stats.failures()) From 6fcec6e4ae8bd60e1f24f0fba78a0c6195301ad0 Mon Sep 17 00:00:00 2001 From: rafael Date: Fri, 10 Jul 2020 15:11:34 +0200 Subject: [PATCH 05/15] fix comments --- reframe/frontend/statistics.py | 123 ++++++++++++++++----------------- reframe/schemas/report.json | 41 ----------- unittests/test_policies.py | 4 +- 3 files changed, 61 insertions(+), 107 deletions(-) delete mode 100644 reframe/schemas/report.json diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 71a597aa2b..b66484f5e2 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -2,14 +2,11 @@ # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause - import json -import os -import reframe import reframe.core.debug as debug import reframe.core.runtime as rt -from reframe.core.exceptions import StatisticsError, ConfigError +from reframe.core.exceptions import format_exception, StatisticsError class TestStats: @@ -17,21 +14,21 @@ class TestStats: def __init__(self): # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] - self._tasks = [[]] + self._alltasks = [[]] def __repr__(self): return debug.repr(self) def add_task(self, task): current_run = rt.runtime().current_run - if current_run == len(self._tasks): - self._tasks.append([]) + if current_run == len(self._alltasks): + self._alltasks.append([]) - self._tasks[current_run].append(task) + self._alltasks[current_run].append(task) def tasks(self, run=-1): try: - return self._tasks[run] + return self._alltasks[run] except IndexError: raise StatisticsError('no such run: %s' % run) from None @@ -52,7 +49,7 @@ def retry_report(self): report.append(line_width * '-') messages = {} - for run in range(1, len(self._tasks)): + for run in range(1, len(self._alltasks)): for t in self.tasks(run): partition_name = '' environ_name = '' @@ -74,21 +71,20 @@ def retry_report(self): return '\n'.join(report) - @property - def report_dict(self): - _report = [] + def json(self): + records = [] current_run = rt.runtime().current_run - for tl, tr in zip(self._tasks, range(current_run + 1)): - for t in tl: + for run_no, run in enumerate(self._alltasks): + for t in run: check = t.check partition = check.current_partition - partname = partition.fullname if partition else None + partfullname = partition.fullname if partition else None environ_name = (check.current_environ.name if check.current_environ else None) - report = { - 'name': check.name, + entry = { + 'testname': check.name, 'description': check.descr, - 'system': partname, + 'system': partfullname, 'environment': environ_name, 'tags': list(check.tags), 'maintainers': check.maintainers, @@ -102,75 +98,74 @@ def report_dict(self): 'failing_reason': None, 'failing_phase': None, 'outputdir': None, - 'stagedir': None + 'stagedir': None, + 'job_stdout': None, + 'job_stderr': None } if check.job: - report['scheduler'] = check.job.scheduler.registered_name - report['jobid'] = check.job.jobid - report['nodelist'] = (check.job.nodelist - if check.job.nodelist - else []) - report['job_stdout'] = check.job.stdout - report['job_stderr'] = check.job.stderr + entry['scheduler'] = partition.scheduler.registered_name + entry['jobid'] = check.job.jobid + entry['nodelist'] = check.job.nodelist or [] + entry['job_stdout'] = check.stdout.evaluate() + entry['job_stderr'] = check.stderr.evaluate() if check._build_job: - report['build_stdout'] = check._build_job.stdout - report['build_stderr'] = check._build_job.stderr + entry['build_stdout'] = check.build_stdout.evaluate() + entry['build_stderr'] = check.build_stderr.evaluate() if t.failed: - report['result'] = 'fail' + entry['result'] = 'fail' if t.exc_info is not None: - from reframe.core.exceptions import format_exception - - report['failing_reason'] = format_exception( + entry['failing_reason'] = format_exception( *t.exc_info) - report['failing_phase'] = t.failed_stage - report['stagedir'] = check.stagedir + entry['failing_phase'] = t.failed_stage + entry['stagedir'] = check.stagedir else: - report['result'] = 'success' - report['outputdir'] = check.outputdir + entry['result'] = 'success' + entry['outputdir'] = check.outputdir - report['try'] = tr + entry['run_no'] = run_no - _report.append(report) + records.append(entry) - return _report + return records def json_report(self): with open('report.json', 'w') as fp: - json.dump(self.report_dict, fp, indent=4) + json.dump(self.json(), fp, indent=4) def failure_report(self): line_width = 78 report = [line_width * '='] report.append('SUMMARY OF FAILURES') - current_run = rt.runtime().current_run - for tf in (t for t in self.report_dict - if t['result'] == 'fail' and t['try'] == current_run): - retry_info = ('(for the last of %s retries)' % current_run - if 'retries' in tf.keys() else '') - + last_run = rt.runtime().current_run + for r in self.json(): + if r['result'] == 'success' or r['run_no'] != last_run: + continue + retry_info = ('(for the last of %s retries)' % last_run + if last_run > 0 else '') report.append(line_width * '-') - report.append('FAILURE INFO for %s %s' % (tf['name'], retry_info)) - report.append(' * Test Description: %s' % tf['description']) - report.append(' * System partition: %s' % tf['system']) - report.append(' * Environment: %s' % tf['environment']) - report.append(' * Stage directory: %s' % tf['stagedir']) + report.append('FAILURE INFO for %s %s' % (r['testname'], + retry_info)) + report.append(' * Test Description: %s' % r['description']) + report.append(' * System partition: %s' % r['system']) + report.append(' * Environment: %s' % r['environment']) + report.append(' * Stage directory: %s' % r['stagedir']) report.append(' * Node list: %s' % - (','.join(tf['nodelist']) - if tf['nodelist'] else None)) - job_type = 'local' if tf['scheduler'] == 'local' else 'batch job' - jobid = tf['jobid'] + (','.join(r['nodelist']) + if r['nodelist'] else None)) + job_type = 'local' if r['scheduler'] == 'local' else 'batch job' + jobid = r['jobid'] report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) - report.append(' * Maintainers: %s' % tf['maintainers']) - report.append(' * Failing phase: %s' % tf['failing_phase']) + report.append(' * Maintainers: %s' % r['maintainers']) + report.append(' * Failing phase: %s' % r['failing_phase']) report.append(" * Rerun with '-n %s -p %s --system %s'" % - (tf['name'], tf['environment'], tf['system'])) - report.append(" * Reason: %s" % tf['failing_reason']) + (r['testname'], r['environment'], r['system'])) + report.append(" * Reason: %s" % r['failing_reason']) - if tf['failing_phase'] == 'sanity': + if r['failing_phase'] == 'sanity': report.append('Sanity check failure') - elif tf['failing_phase'] == 'performance': + elif r['failing_phase'] == 'performance': report.append('Performance check failure') else: # This shouldn't happen... @@ -185,10 +180,10 @@ def failure_stats(self): for tf in (t for t in self.tasks(current_run) if t.failed): check = tf.check partition = check.current_partition - partname = partition.fullname if partition else 'None' + partfullname = partition.fullname if partition else 'None' environ_name = (check.current_environ.name if check.current_environ else 'None') - f = f'[{check.name}, {environ_name}, {partname}]' + f = f'[{check.name}, {environ_name}, {partfullname}]' if tf.failed_stage not in failures: failures[tf.failed_stage] = [] diff --git a/reframe/schemas/report.json b/reframe/schemas/report.json deleted file mode 100644 index a9be4ce9f9..0000000000 --- a/reframe/schemas/report.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "type": "array", - "items": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "description": {"type": ["string", "null"]}, - "system": {"type": ["string", "null"]}, - "environment": {"type": ["string", "null"]}, - "stagedir": {"type": ["string", "null"]}, - "outputdir": {"type": ["string", "null"]}, - "nodelist": { - "type": "array", - "items": {"type": ["string"]} - }, - "scheduler": {"type": ["string", "null"]}, - "jobid": {"type": ["number", "null"]}, - "result": {"type": "string"}, - "failing_phase": {"type": ["string", "null"]}, - "failing_reason": {"type": ["string", "null"]}, - "build_stdout": {"type": ["string", "null"]}, - "build_stderr": {"type": ["string", "null"]}, - "job_stdout": {"type": ["string", "null"]}, - "job_stderr": {"type": ["string", "null"]}, - "maintainers": { - "type": "array", - "items": {"type": ["string", "null"]} - }, - "tags": { - "type": "array", - "items": {"type": ["string", "null"]} - }, - "try": {"type": "number"} - }, - "additionalProperties": false, - "required": ["name", "description", "system", "environment", "stagedir", - "outputdir", "nodelist", "scheduler", "jobid", "result", - "failing_phase", "failing_reason", "build_stdout", "build_stderr", - "job_stdout", "job_stderr", "maintainers", "tags", "try"] - } -} diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 0f04f52162..79b13929b7 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -110,7 +110,7 @@ def num_failures_stage(runner, stage): def open_json_schema(): # Open and store the JSON schema for later validation schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe', - 'schemas', 'report.json') + 'schemas', 'runreport.json') with open(schema_filename) as fp: try: schema = json.loads(fp.read()) @@ -126,7 +126,7 @@ def test_runall(make_runner, make_cases, common_exec_ctx): runner = make_runner() runner.runall(make_cases()) stats = runner.stats - json_out = runner.stats.report_dict + json_out = runner.stats.json() schema = open_json_schema() jsonschema.validate(json_out, schema) assert 8 == stats.num_cases() From f905493680b328015a262eb0e27c8122cd781145 Mon Sep 17 00:00:00 2001 From: rafael Date: Mon, 13 Jul 2020 13:00:54 +0200 Subject: [PATCH 06/15] remove exception handling --- unittests/test_policies.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 79b13929b7..c75baefac6 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -112,12 +112,7 @@ def open_json_schema(): schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe', 'schemas', 'runreport.json') with open(schema_filename) as fp: - try: - schema = json.loads(fp.read()) - except json.JSONDecodeError as e: - raise ReframeFatalError( - f"invalid configuration schema: '{schema_filename}'" - ) from e + schema = json.loads(fp.read()) return schema From cc2f903cf53ad3eb957c2021871c0a9993c09e71 Mon Sep 17 00:00:00 2001 From: rafael Date: Mon, 13 Jul 2020 13:22:29 +0200 Subject: [PATCH 07/15] add validate_report --- unittests/test_policies.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index c75baefac6..1b305abd11 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -109,21 +109,23 @@ def num_failures_stage(runner, stage): def open_json_schema(): # Open and store the JSON schema for later validation - schema_filename = os.path.join(reframe.INSTALL_PREFIX, 'reframe', - 'schemas', 'runreport.json') + schema_filename = os.path.join('reframe/schemas/runreport.json') with open(schema_filename) as fp: schema = json.loads(fp.read()) return schema +def validate_report(runreport): + schema = open_json_schema() + jsonschema.validate(runreport, schema) + + def test_runall(make_runner, make_cases, common_exec_ctx): runner = make_runner() runner.runall(make_cases()) stats = runner.stats - json_out = runner.stats.json() - schema = open_json_schema() - jsonschema.validate(json_out, schema) + validate_report(runner.stats.json()) assert 8 == stats.num_cases() assert_runall(runner) assert 5 == len(stats.failures()) From 30a459cd01b25dd2266e024c5dde070d6a0a19e0 Mon Sep 17 00:00:00 2001 From: rafael Date: Wed, 15 Jul 2020 16:20:30 +0200 Subject: [PATCH 08/15] add timing to report --- reframe/frontend/statistics.py | 57 ++++++++++++++++++++-------------- reframe/schemas/runreport.json | 53 +++++++++++++++++++++++++++++++ unittests/test_policies.py | 8 +---- 3 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 reframe/schemas/runreport.json diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index b66484f5e2..8042be73a0 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -78,14 +78,11 @@ def json(self): for t in run: check = t.check partition = check.current_partition - partfullname = partition.fullname if partition else None - environ_name = (check.current_environ.name - if check.current_environ else None) entry = { 'testname': check.name, 'description': check.descr, - 'system': partfullname, - 'environment': environ_name, + 'system': None, + 'environment': None, 'tags': list(check.tags), 'maintainers': check.maintainers, 'scheduler': None, @@ -100,10 +97,24 @@ def json(self): 'outputdir': None, 'stagedir': None, 'job_stdout': None, - 'job_stderr': None + 'job_stderr': None, + 'time_setup': t.duration('setup'), + 'time_compile': t.duration('complete'), + 'time_run': t.duration('t.duration'), + 'time_sanity': t.duration('sanity'), + 'time_performance': t.duration('performance'), + 'time_total': t.duration('total') } - if check.job: + partition, environ = (check.current_partition, + check.current_environ) + if check.current_partition: + entry['system'] = partition.fullname entry['scheduler'] = partition.scheduler.registered_name + + if check.current_environ: + entry['environment'] = environ.name + + if check.job: entry['jobid'] = check.job.jobid entry['nodelist'] = check.job.nodelist or [] entry['job_stdout'] = check.stdout.evaluate() @@ -142,27 +153,25 @@ def failure_report(self): for r in self.json(): if r['result'] == 'success' or r['run_no'] != last_run: continue - retry_info = ('(for the last of %s retries)' % last_run + + retry_info = (f'(for the last of {last_run} retries)' if last_run > 0 else '') report.append(line_width * '-') - report.append('FAILURE INFO for %s %s' % (r['testname'], - retry_info)) - report.append(' * Test Description: %s' % r['description']) - report.append(' * System partition: %s' % r['system']) - report.append(' * Environment: %s' % r['environment']) - report.append(' * Stage directory: %s' % r['stagedir']) - report.append(' * Node list: %s' % - (','.join(r['nodelist']) - if r['nodelist'] else None)) + report.append(f"FAILURE INFO for {r['testname']} {retry_info}") + report.append(f" * Test Description: {r['description']}") + report.append(f" * System partition: {r['system']}") + report.append(f" * Environment: {r['environment']}") + report.append(f" * Stage directory: {r['stagedir']}") + nodelist = ','.join(r['nodelist']) if r['nodelist'] else None + report.append(f" * Node list: {nodelist}") job_type = 'local' if r['scheduler'] == 'local' else 'batch job' jobid = r['jobid'] - report.append(' * Job type: %s (id=%s)' % (job_type, jobid)) - report.append(' * Maintainers: %s' % r['maintainers']) - report.append(' * Failing phase: %s' % r['failing_phase']) - report.append(" * Rerun with '-n %s -p %s --system %s'" % - (r['testname'], r['environment'], r['system'])) - report.append(" * Reason: %s" % r['failing_reason']) - + report.append(f" * Job type: {job_type} (id={r['jobid']})") + report.append(f" * Maintainers: {r['maintainers']}") + report.append(f" * Failing phase: {r['failing_phase']}") + report.append(f" * Rerun with '-n {r['testname']}" + f" -p {r['environment']} --system {r['system']}'") + report.append(f" * Reason: {r['failing_reason']}") if r['failing_phase'] == 'sanity': report.append('Sanity check failure') elif r['failing_phase'] == 'performance': diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json new file mode 100644 index 0000000000..2abb3fc525 --- /dev/null +++ b/reframe/schemas/runreport.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://raw.githubusercontent.com/eth-cscs/reframe/master/schemas/runreport.json", + "title": "Validation schema for ReFrame's run report", + "type": "array", + "items": { + "type": "object", + "properties": { + "testname": {"type": "string"}, + "description": {"type": ["string"]}, + "system": {"type": ["string", "null"]}, + "environment": {"type": ["string", "null"]}, + "stagedir": {"type": ["string", "null"]}, + "outputdir": {"type": ["string", "null"]}, + "nodelist": { + "type": "array", + "items": {"type": "string"} + }, + "scheduler": {"type": ["string", "null"]}, + "jobid": {"type": ["number", "null"]}, + "result": {"type": "string"}, + "failing_phase": {"type": ["string", "null"]}, + "failing_reason": {"type": ["string", "null"]}, + "build_stdout": {"type": ["string", "null"]}, + "build_stderr": {"type": ["string", "null"]}, + "job_stdout": {"type": ["string", "null"]}, + "job_stderr": {"type": ["string", "null"]}, + "maintainers": { + "type": "array", + "items": {"type": ["string", "null"]} + }, + "tags": { + "type": "array", + "items": {"type": ["string", "null"]} + }, + "run_no": {"type": "number"}, + "time_setup": {"type": "number"}, + "time_compile": {"type": ["number", "null"]}, + "time_run": {"type": ["number", "null"]}, + "time_sanity": {"type": ["number", "null"]}, + "time_performance": {"type": ["number", "null"]}, + "time_total": {"type": "number"} + }, + "additionalProperties": false, + "required": ["build_stderr", "build_stdout", "description", + "environment", "failing_phase", "failing_reason", + "job_stderr", "job_stdout", "jobid", "maintainers", + "nodelist", "outputdir", "result", "run_no", + "scheduler", "stagedir", "system", "tags", "testname", + "time_compile", "time_performance", "time_run", + "time_sanity", "time_setup", "time_total"] + } +} diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 1b305abd11..21c01b6835 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -107,17 +107,11 @@ def num_failures_stage(runner, stage): return len([t for t in stats.failures() if t.failed_stage == stage]) -def open_json_schema(): - # Open and store the JSON schema for later validation +def validate_report(runreport): schema_filename = os.path.join('reframe/schemas/runreport.json') with open(schema_filename) as fp: schema = json.loads(fp.read()) - return schema - - -def validate_report(runreport): - schema = open_json_schema() jsonschema.validate(runreport, schema) From 6cf2ddea897bc4a355222746387875a02f5db040 Mon Sep 17 00:00:00 2001 From: rafael Date: Thu, 16 Jul 2020 16:20:20 +0200 Subject: [PATCH 09/15] fix comments --- reframe/core/systems.py | 4 ++-- reframe/frontend/cli.py | 14 +++++++++++++- reframe/frontend/statistics.py | 35 +++++++++++++++++----------------- reframe/schemas/runreport.json | 19 ++++++++++-------- reframe/utility/__init__.py | 18 +++++++++++++++++ 5 files changed, 61 insertions(+), 29 deletions(-) diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 2e67b6904b..6058232868 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -189,12 +189,12 @@ def json(self): 'container_platforms': [ { 'type': ctype, - 'modules': [m.name for m in cpenv.modules], + 'modules': [m for m in cpenv.modules], 'variables': [[n, v] for n, v in cpenv.variables.items()] } for ctype, cpenv in self._container_environs.items() ], - 'modules': [m.name for m in self._local_env.modules], + 'modules': [m for m in self._local_env.modules], 'variables': [[n, v] for n, v in self._local_env.variables.items()], 'environs': [e.name for e in self._environs], diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d81c4fe0f3..d4dddeaa7a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -30,6 +30,7 @@ AsynchronousExecutionPolicy) from reframe.frontend.loader import RegressionCheckLoader from reframe.frontend.printer import PrettyPrinter +from reframe.utility import get_next_runreport_index def format_check(check, detailed): @@ -136,6 +137,10 @@ def main(): help='Save ReFrame log files to the output directory', envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) + output_options.add_argument( + '--keep-runreport', action='store_true', default=False, + help="Do not overwrite runreport.json", + ) # Check discovery options locate_options.add_argument( @@ -700,7 +705,14 @@ def print_infoline(param, value): if options.performance_report: printer.info(runner.stats.performance_report()) - runner.stats.json_report() + if options.keep_runreport: + runreport_id = get_next_runreport_index() + runreport_name = f'runreport-{runreport_id}.json' + else: + runreport_name = 'runreport.json' + + with open(runreport_name, 'w') as fp: + json.dump(runner.stats.json(), fp, indent=4) else: printer.error("No action specified. Please specify `-l'/`-L' for " diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 8042be73a0..a3305ee3cf 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -2,7 +2,6 @@ # ReFrame Project Developers. See the top-level LICENSE file for details. # # SPDX-License-Identifier: BSD-3-Clause -import json import reframe.core.debug as debug import reframe.core.runtime as rt @@ -15,6 +14,7 @@ class TestStats: def __init__(self): # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] self._alltasks = [[]] + self._records = [] def __repr__(self): return debug.repr(self) @@ -71,8 +71,11 @@ def retry_report(self): return '\n'.join(report) - def json(self): - records = [] + def json(self, force=False): + if not force and self._records: + return self._records + + self._records = [] current_run = rt.runtime().current_run for run_no, run in enumerate(self._alltasks): for t in run: @@ -81,7 +84,7 @@ def json(self): entry = { 'testname': check.name, 'description': check.descr, - 'system': None, + 'system': check.current_system.name, 'environment': None, 'tags': list(check.tags), 'maintainers': check.maintainers, @@ -99,19 +102,19 @@ def json(self): 'job_stdout': None, 'job_stderr': None, 'time_setup': t.duration('setup'), - 'time_compile': t.duration('complete'), - 'time_run': t.duration('t.duration'), + 'time_compile': t.duration('complete_complete'), + 'time_run': t.duration('run_complete'), 'time_sanity': t.duration('sanity'), 'time_performance': t.duration('performance'), 'time_total': t.duration('total') } - partition, environ = (check.current_partition, - check.current_environ) - if check.current_partition: + partition = check.current_partition + environ = check.current_environ + if partition: entry['system'] = partition.fullname entry['scheduler'] = partition.scheduler.registered_name - if check.current_environ: + if environ: entry['environment'] = environ.name if check.job: @@ -135,15 +138,11 @@ def json(self): entry['result'] = 'success' entry['outputdir'] = check.outputdir - entry['run_no'] = run_no - - records.append(entry) + entry['runid'] = run_no - return records + self._records.append(entry) - def json_report(self): - with open('report.json', 'w') as fp: - json.dump(self.json(), fp, indent=4) + return self._records def failure_report(self): line_width = 78 @@ -151,7 +150,7 @@ def failure_report(self): report.append('SUMMARY OF FAILURES') last_run = rt.runtime().current_run for r in self.json(): - if r['result'] == 'success' or r['run_no'] != last_run: + if r['result'] == 'success' or r['runid'] != last_run: continue retry_info = (f'(for the last of {last_run} retries)' diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 2abb3fc525..0f708537fa 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -8,7 +8,7 @@ "properties": { "testname": {"type": "string"}, "description": {"type": ["string"]}, - "system": {"type": ["string", "null"]}, + "system": {"type": "string"}, "environment": {"type": ["string", "null"]}, "stagedir": {"type": ["string", "null"]}, "outputdir": {"type": ["string", "null"]}, @@ -18,7 +18,10 @@ }, "scheduler": {"type": ["string", "null"]}, "jobid": {"type": ["number", "null"]}, - "result": {"type": "string"}, + "result": { + "type": "string", + "enum": ["success", "fail"] + }, "failing_phase": {"type": ["string", "null"]}, "failing_reason": {"type": ["string", "null"]}, "build_stdout": {"type": ["string", "null"]}, @@ -27,25 +30,25 @@ "job_stderr": {"type": ["string", "null"]}, "maintainers": { "type": "array", - "items": {"type": ["string", "null"]} + "items": {"type": ["string"]} }, "tags": { "type": "array", - "items": {"type": ["string", "null"]} + "items": {"type": ["string"]} }, - "run_no": {"type": "number"}, - "time_setup": {"type": "number"}, + "runid": {"type": "number"}, + "time_setup": {"type": ["number", "null"]}, "time_compile": {"type": ["number", "null"]}, "time_run": {"type": ["number", "null"]}, "time_sanity": {"type": ["number", "null"]}, "time_performance": {"type": ["number", "null"]}, - "time_total": {"type": "number"} + "time_total": {"type": ["number", "null"]} }, "additionalProperties": false, "required": ["build_stderr", "build_stdout", "description", "environment", "failing_phase", "failing_reason", "job_stderr", "job_stdout", "jobid", "maintainers", - "nodelist", "outputdir", "result", "run_no", + "nodelist", "outputdir", "result", "runid", "scheduler", "stagedir", "system", "tags", "testname", "time_compile", "time_performance", "time_run", "time_sanity", "time_setup", "time_total"] diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index b16a0632ca..c14aff0e3d 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -16,6 +16,24 @@ from collections import UserDict +def get_next_runreport_index(): + runreports = os.listdir('.') + pattern = r'runreport-\d+.json' + filenames = filter(re.compile(pattern).match, runreports) + filenames = sorted(filenames) + + if not filenames: + return 1 + + for i, f in enumerate(filenames, 1): + idx = f.split('.')[0].split('-')[1] + idx = int(idx) + if idx != i: + return i + + return idx + 1 + + def seconds_to_hms(seconds): '''Convert time in seconds to a tuple of ``(hour, minutes, seconds)``.''' From 82d0fba7769b313e178b634c4f78874e74bc8e60 Mon Sep 17 00:00:00 2001 From: rafael Date: Tue, 21 Jul 2020 18:03:11 +0200 Subject: [PATCH 10/15] new runreport format --- reframe/frontend/cli.py | 62 +++++++++++---- reframe/frontend/statistics.py | 16 ++-- reframe/schemas/runreport.json | 138 +++++++++++++++++++++------------ reframe/utility/__init__.py | 11 ++- unittests/test_policies.py | 20 ++++- 5 files changed, 168 insertions(+), 79 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index d4dddeaa7a..11a043ebd1 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +import datetime import inspect import json import os @@ -10,6 +11,7 @@ import socket import sys import traceback +from pathlib import Path import reframe import reframe.core.config as config @@ -138,7 +140,7 @@ def main(): envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) output_options.add_argument( - '--keep-runreport', action='store_true', default=False, + '--runreport-name', action='store', metavar='NAME', help="Do not overwrite runreport.json", ) @@ -502,19 +504,32 @@ def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") + reframe_info = { + 'version': os_ext.reframe_version(), + 'command': repr(' '.join(sys.argv)), + 'user': f"{os_ext.osuser() or ''}", + 'host': socket.gethostname(), + 'working_directory': repr(os.getcwd()), + 'check_search_path': f"{':'.join(loader.load_path)!r}", + 'recursive_search_path': loader.recurse, + 'settings_file': site_config.filename, + 'stage_prefix': repr(rt.stage_prefix), + 'output_prefix': repr(rt.output_prefix), + } + # Print command line printer.info(f"[ReFrame Setup]") - print_infoline('version', os_ext.reframe_version()) - print_infoline('command', repr(' '.join(sys.argv))) + print_infoline('version', reframe_info['version']) + print_infoline('command', reframe_info['command']) print_infoline('launched by', - f"{os_ext.osuser() or ''}@{socket.gethostname()}") - print_infoline('working directory', repr(os.getcwd())) - print_infoline('settings file', f'{site_config.filename!r}') + f"{reframe_info['user']}@{reframe_info['host']}") + print_infoline('working directory', reframe_info['working_directory']) + print_infoline('settings file', f"{reframe_info['settings_file']!r}") print_infoline('check search path', - f"{'(R) ' if loader.recurse else ''}" - f"{':'.join(loader.load_path)!r}") - print_infoline('stage directory', repr(rt.stage_prefix)) - print_infoline('output directory', repr(rt.output_prefix)) + f"{'(R) ' if reframe_info['recursive_search_path'] else ''}" + f"{reframe_info['check_search_path']!r}") + print_infoline('stage directory', reframe_info['stage_prefix']) + print_infoline('output directory', reframe_info['output_prefix']) printer.info('') try: # Locate and load checks @@ -689,9 +704,13 @@ def print_infoline(param, value): max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: + reframe_info['start_time'] = ( + datetime.datetime.today().strftime('%c %Z')) runner.runall(testcases) finally: # Print a retry report if we did any retries + reframe_info['end_time'] = ( + datetime.datetime.today().strftime('%c %Z')) if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) @@ -705,14 +724,23 @@ def print_infoline(param, value): if options.performance_report: printer.info(runner.stats.performance_report()) - if options.keep_runreport: - runreport_id = get_next_runreport_index() - runreport_name = f'runreport-{runreport_id}.json' + if options.runreport_name: + runreport_file = options.runreport_name else: - runreport_name = 'runreport.json' - - with open(runreport_name, 'w') as fp: - json.dump(runner.stats.json(), fp, indent=4) + runreport_dir = os.path.join(Path.home(), '.local/reframe') + runreport_id = get_next_runreport_index(runreport_dir) + runreport_name = f'runreport-{runreport_id}.json' + Path(runreport_dir).mkdir(parents=True, exist_ok=True) + runreport_file = os.path.join(runreport_dir, + runreport_name) + + try: + with open(runreport_file, 'w') as fp: + json.dump(runner.stats.json(reframe_info, force=True), + fp, indent=4) + except OSError: + printer.error(f'invalid path: {runreport_file}') + sys.exit(1) else: printer.error("No action specified. Please specify `-l'/`-L' for " diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index a3305ee3cf..72b4aad274 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -71,13 +71,14 @@ def retry_report(self): return '\n'.join(report) - def json(self, force=False): + def json(self, reframe_info=None, force=False): if not force and self._records: return self._records - self._records = [] + self._records = {'run_info': []} current_run = rt.runtime().current_run for run_no, run in enumerate(self._alltasks): + tests = [] for t in run: check = t.check partition = check.current_partition @@ -138,9 +139,12 @@ def json(self, force=False): entry['result'] = 'success' entry['outputdir'] = check.outputdir - entry['runid'] = run_no + tests.append(entry) + + self._records['run_info'].append({'runid': run_no, 'tests': tests}) - self._records.append(entry) + if reframe_info: + self._records['reframe_info'] = reframe_info return self._records @@ -149,8 +153,8 @@ def failure_report(self): report = [line_width * '='] report.append('SUMMARY OF FAILURES') last_run = rt.runtime().current_run - for r in self.json(): - if r['result'] == 'success' or r['runid'] != last_run: + for r in self.json()['run_info'][last_run]['tests']: + if r['result'] == 'success': continue retry_info = (f'(for the last of {last_run} retries)' diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index 0f708537fa..fff4ceed7a 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -2,55 +2,95 @@ "$schema": "http://json-schema.org/draft-07/schema#", "$id": "https://raw.githubusercontent.com/eth-cscs/reframe/master/schemas/runreport.json", "title": "Validation schema for ReFrame's run report", - "type": "array", - "items": { - "type": "object", - "properties": { - "testname": {"type": "string"}, - "description": {"type": ["string"]}, - "system": {"type": "string"}, - "environment": {"type": ["string", "null"]}, - "stagedir": {"type": ["string", "null"]}, - "outputdir": {"type": ["string", "null"]}, - "nodelist": { - "type": "array", - "items": {"type": "string"} + "type": "object", + "properties": { + "reframe_info": { + "type": "object", + "properties": { + "version": {"type": "string"}, + "command": {"type": "string"}, + "user": {"type": "string"}, + "host": {"type": "string"}, + "check_search_path": {"type": "string"}, + "working_directory": {"type": "string"}, + "recursive_search_path": {"type": "boolean"}, + "settings_file": {"type": "string"}, + "stage_prefix": {"type": "string"}, + "output_prefix": {"type": "string"}, + "start_time": {"type": "string"}, + "end_time": {"type": "string"} }, - "scheduler": {"type": ["string", "null"]}, - "jobid": {"type": ["number", "null"]}, - "result": { - "type": "string", - "enum": ["success", "fail"] - }, - "failing_phase": {"type": ["string", "null"]}, - "failing_reason": {"type": ["string", "null"]}, - "build_stdout": {"type": ["string", "null"]}, - "build_stderr": {"type": ["string", "null"]}, - "job_stdout": {"type": ["string", "null"]}, - "job_stderr": {"type": ["string", "null"]}, - "maintainers": { - "type": "array", - "items": {"type": ["string"]} - }, - "tags": { - "type": "array", - "items": {"type": ["string"]} - }, - "runid": {"type": "number"}, - "time_setup": {"type": ["number", "null"]}, - "time_compile": {"type": ["number", "null"]}, - "time_run": {"type": ["number", "null"]}, - "time_sanity": {"type": ["number", "null"]}, - "time_performance": {"type": ["number", "null"]}, - "time_total": {"type": ["number", "null"]} + "additionalProperties": false, + "required": ["check_search_path", "command", "end_time", "host", + "output_prefix", "recursive_search_path", + "stage_prefix", "start_time", "user", "version", + "working_directory"] }, - "additionalProperties": false, - "required": ["build_stderr", "build_stdout", "description", - "environment", "failing_phase", "failing_reason", - "job_stderr", "job_stdout", "jobid", "maintainers", - "nodelist", "outputdir", "result", "runid", - "scheduler", "stagedir", "system", "tags", "testname", - "time_compile", "time_performance", "time_run", - "time_sanity", "time_setup", "time_total"] - } + "run_info": { + "type": "array", + "items": { + "type": "object", + "properties": { + "runid": {"type": "number"}, + "tests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "testname": {"type": "string"}, + "description": {"type": "string"}, + "system": {"type": "string"}, + "environment": {"type": ["string", "null"]}, + "stagedir": {"type": ["string", "null"]}, + "outputdir": {"type": ["string", "null"]}, + "nodelist": { + "type": "array", + "items": {"type": "string"} + }, + "scheduler": {"type": ["string", "null"]}, + "jobid": {"type": ["number", "null"]}, + "result": { + "type": "string", + "enum": ["success", "fail"] + }, + "failing_phase": {"type": ["string", "null"]}, + "failing_reason": {"type": ["string", "null"]}, + "build_stdout": {"type": ["string", "null"]}, + "build_stderr": {"type": ["string", "null"]}, + "job_stdout": {"type": ["string", "null"]}, + "job_stderr": {"type": ["string", "null"]}, + "maintainers": { + "type": "array", + "items": {"type": ["string"]} + }, + "tags": { + "type": "array", + "items": {"type": "string"} + }, + "runid": {"type": "number"}, + "time_setup": {"type": ["number", "null"]}, + "time_compile": {"type": ["number", "null"]}, + "time_run": {"type": ["number", "null"]}, + "time_sanity": {"type": ["number", "null"]}, + "time_performance": {"type": ["number", "null"]}, + "time_total": {"type": ["number", "null"]} + }, + "additionalProperties": false, + "required": ["build_stderr", "build_stdout", "description", + "environment", "failing_phase", "failing_reason", + "job_stderr", "job_stdout", "jobid", "maintainers", + "nodelist", "outputdir", "result", "scheduler", + "stagedir", "system", "tags", "testname", + "time_compile", "time_performance", "time_run", + "time_sanity", "time_setup", "time_total"] + }, + "additionalProperties": false, + "required": ["runid", "tests"] + } + } + } + } + }, + "additionalProperties": false, + "required": ["reframe_info", "run_info"] } diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index c14aff0e3d..02cc53b952 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -16,18 +16,17 @@ from collections import UserDict -def get_next_runreport_index(): - runreports = os.listdir('.') +def get_next_runreport_index(runreport_dir): + runreports = os.listdir(runreport_dir) pattern = r'runreport-\d+.json' filenames = filter(re.compile(pattern).match, runreports) - filenames = sorted(filenames) if not filenames: return 1 - for i, f in enumerate(filenames, 1): - idx = f.split('.')[0].split('-')[1] - idx = int(idx) + idx_list = sorted([int(f.split('.')[0].split('-')[1]) + for f in filenames]) + for i, idx in enumerate(idx_list): if idx != i: return i diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 21c01b6835..17e823597d 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,10 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause +import datetime import json import jsonschema import os import pytest +import socket +import sys import reframe import reframe.core.runtime as rt @@ -116,10 +119,25 @@ def validate_report(runreport): def test_runall(make_runner, make_cases, common_exec_ctx): + reframe_info = { + 'version': os_ext.reframe_version(), + 'command': repr(' '.join(sys.argv)), + 'user': f"{os_ext.osuser() or ''}", + 'host': socket.gethostname(), + 'working_directory': repr(os.getcwd()), + 'check_search_path': 'unittests/resources/checks', + 'recursive_search_path': True, + 'settings_file': fixtures.TEST_CONFIG_FILE, + 'stage_prefix': repr(rt.runtime().stage_prefix), + 'output_prefix': repr(rt.runtime().output_prefix), + } runner = make_runner() + reframe_info['start_time'] = datetime.datetime.today().strftime('%c %Z') runner.runall(make_cases()) + reframe_info['end_time'] = datetime.datetime.today().strftime('%c %Z') stats = runner.stats - validate_report(runner.stats.json()) + runreport = runner.stats.json(reframe_info, force=True) + validate_report(runreport) assert 8 == stats.num_cases() assert_runall(runner) assert 5 == len(stats.failures()) From 9705d69355f7e3e9071a5dfd38d06b62868d2dfb Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 22 Jul 2020 22:32:25 +0200 Subject: [PATCH 11/15] Fix and expand JSON report --- reframe/core/pipeline.py | 4 ++ reframe/frontend/cli.py | 120 +++++++++++++++++++------------ reframe/frontend/statistics.py | 125 +++++++++++++++++++-------------- reframe/schemas/config.json | 7 +- reframe/schemas/runreport.json | 121 +++++++++++++++++-------------- reframe/utility/__init__.py | 17 ----- unittests/test_cli.py | 3 + unittests/test_policies.py | 52 ++++++++------ 8 files changed, 260 insertions(+), 189 deletions(-) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index a17205113c..b30ae0c860 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -925,6 +925,10 @@ def stderr(self): ''' return self._job.stderr + @property + def build_job(self): + return self._build_job + @property @sn.sanity_function def build_stdout(self): diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 56bbd92e05..bbee0720f9 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -3,15 +3,14 @@ # # SPDX-License-Identifier: BSD-3-Clause -import datetime import inspect import json import os import re import socket import sys +import time import traceback -from pathlib import Path import reframe import reframe.core.config as config @@ -32,7 +31,6 @@ AsynchronousExecutionPolicy) from reframe.frontend.loader import RegressionCheckLoader from reframe.frontend.printer import PrettyPrinter -from reframe.utility import get_next_runreport_index def format_check(check, detailed): @@ -72,6 +70,22 @@ def list_checks(checks, printer, detailed=False): printer.info('\nFound %d check(s).' % len(checks)) +def generate_report_filename(filepatt): + if not '{sessionid}' in filepatt: + return filepatt + + search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') + new_id = -1 + for filename in os.listdir(os.path.dirname(filepatt)): + match = re.match(search_patt, filename) + if match: + found_id = int(match.group(1)) + new_id = max(found_id, new_id) + + new_id += 1 + return filepatt.format(sessionid=new_id) + + def main(): # Setup command line options argparser = argparse.ArgumentParser() @@ -140,8 +154,10 @@ def main(): envvar='RFM_SAVE_LOG_FILES', configvar='general/save_log_files' ) output_options.add_argument( - '--runreport-name', action='store', metavar='NAME', - help="Do not overwrite runreport.json", + '--report-file', action='store', metavar='FILE', + help="Store JSON run report in FILE", + envvar='RFM_REPORT_FILE', + configvar='general/report_file' ) # Check discovery options @@ -516,32 +532,33 @@ def print_infoline(param, value): param = param + ':' printer.info(f" {param.ljust(18)} {value}") - reframe_info = { + session_info = { + 'cmdline': ' '.join(sys.argv), + 'config_file': rt.site_config.filename, + 'data_version': '1.0', + 'hostname': socket.gethostname(), + 'prefix_output': rt.output_prefix, + 'prefix_stage': rt.stage_prefix, + 'user': os_ext.osuser(), 'version': os_ext.reframe_version(), - 'command': repr(' '.join(sys.argv)), - 'user': f"{os_ext.osuser() or ''}", - 'host': socket.gethostname(), - 'working_directory': repr(os.getcwd()), - 'check_search_path': f"{':'.join(loader.load_path)!r}", - 'recursive_search_path': loader.recurse, - 'settings_file': site_config.filename, - 'stage_prefix': repr(rt.stage_prefix), - 'output_prefix': repr(rt.output_prefix), + 'workdir': os.getcwd(), } # Print command line printer.info(f"[ReFrame Setup]") - print_infoline('version', reframe_info['version']) - print_infoline('command', reframe_info['command']) - print_infoline('launched by', - f"{reframe_info['user']}@{reframe_info['host']}") - print_infoline('working directory', reframe_info['working_directory']) - print_infoline('settings file', f"{reframe_info['settings_file']!r}") + print_infoline('version', session_info['version']) + print_infoline('command', session_info['cmdline']) + print_infoline( + f"launched by", + f"{session_info['user'] or ''}@{session_info['hostname']}" + ) + print_infoline('working directory', repr(session_info['workdir'])) + print_infoline('settings file', f"{session_info['config_file']!r}") print_infoline('check search path', - f"{'(R) ' if reframe_info['recursive_search_path'] else ''}" - f"{reframe_info['check_search_path']!r}") - print_infoline('stage directory', reframe_info['stage_prefix']) - print_infoline('output directory', reframe_info['output_prefix']) + f"{'(R) ' if loader.recurse else ''}" + f"{':'.join(loader.load_path)!r}") + print_infoline('stage directory', repr(session_info['prefix_stage'])) + print_infoline('output directory', repr(session_info['prefix_output'])) printer.info('') try: # Locate and load checks @@ -716,13 +733,19 @@ def print_infoline(param, value): max_retries) from None runner = Runner(exec_policy, printer, max_retries) try: - reframe_info['start_time'] = ( - datetime.datetime.today().strftime('%c %Z')) + time_start = time.time() + session_info['time_start'] = time.strftime( + '%FT%T%z', time.localtime(time_start), + ) runner.runall(testcases) finally: + time_end = time.time() + session_info['time_end'] = time.strftime( + '%FT%T%z', time.localtime(time_end) + ) + session_info['time_elapsed'] = time_end - time_start + # Print a retry report if we did any retries - reframe_info['end_time'] = ( - datetime.datetime.today().strftime('%c %Z')) if runner.stats.failures(run=0): printer.info(runner.stats.retry_report()) @@ -736,23 +759,30 @@ def print_infoline(param, value): if options.performance_report: printer.info(runner.stats.performance_report()) - if options.runreport_name: - runreport_file = options.runreport_name - else: - runreport_dir = os.path.join(Path.home(), '.local/reframe') - runreport_id = get_next_runreport_index(runreport_dir) - runreport_name = f'runreport-{runreport_id}.json' - Path(runreport_dir).mkdir(parents=True, exist_ok=True) - runreport_file = os.path.join(runreport_dir, - runreport_name) - + # Generate the report for this session + report_file = os.path.normpath( + os_ext.expandvars(rt.get_option('general/0/report_file')) + ) + os.makedirs(os.path.dirname(report_file), exist_ok=True) + + # Build final report JSON + run_stats = runner.stats.json() + session_info.update({ + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'] + }) + json_report = { + 'session_info': session_info, + 'runs': run_stats + } + report_file = generate_report_filename(report_file) try: - with open(runreport_file, 'w') as fp: - json.dump(runner.stats.json(reframe_info, force=True), - fp, indent=4) - except OSError: - printer.error(f'invalid path: {runreport_file}') - sys.exit(1) + with open(report_file, 'w') as fp: + json.dump(json_report, fp, indent=2) + except OSError as e: + printer.warning( + f'failed to generate report in {report_file!r}: {e}' + ) else: printer.error("No action specified. Please specify `-l'/`-L' for " diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index f2b115ff86..ae7ece0f43 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -13,7 +13,9 @@ class TestStats: def __init__(self): # Tasks per run stored as follows: [[run0_tasks], [run1_tasks], ...] self._alltasks = [[]] - self._records = [] + + # Data collected for all the runs of this session in JSON format + self._run_data = [] def add_task(self, task): current_run = rt.runtime().current_run @@ -44,7 +46,6 @@ def retry_report(self): report.append('SUMMARY OF RETRIES') report.append(line_width * '-') messages = {} - for run in range(1, len(self._alltasks)): for t in self.tasks(run): partition_name = '' @@ -67,42 +68,41 @@ def retry_report(self): return '\n'.join(report) - def json(self, reframe_info=None, force=False): - if not force and self._records: - return self._records + def json(self, force=False): + if not force and self._run_data: + return self._run_data - self._records = {'run_info': []} - current_run = rt.runtime().current_run - for run_no, run in enumerate(self._alltasks): - tests = [] + for runid, run in enumerate(self._alltasks): + testcases = [] + num_failures = 0 for t in run: check = t.check partition = check.current_partition entry = { - 'testname': check.name, + 'build_stderr': None, + 'build_stdout': None, 'description': check.descr, - 'system': check.current_system.name, 'environment': None, - 'tags': list(check.tags), - 'maintainers': check.maintainers, - 'scheduler': None, + 'fail_reason': None, + 'fail_phase': None, 'jobid': None, - 'nodelist': [], - 'job_stdout': None, 'job_stderr': None, - 'build_stdout': None, - 'build_stderr': None, - 'failing_reason': None, - 'failing_phase': None, + 'job_stdout': None, + 'name': check.name, + 'maintainers': check.maintainers, + 'nodelist': [], 'outputdir': None, + 'perfvars': None, + 'result': None, 'stagedir': None, - 'job_stdout': None, - 'job_stderr': None, - 'time_setup': t.duration('setup'), - 'time_compile': t.duration('complete_complete'), + 'scheduler': None, + 'system': check.current_system.name, + 'tags': list(check.tags), + 'time_compile': t.duration('compile_complete'), + 'time_performance': t.duration('performance'), 'time_run': t.duration('run_complete'), 'time_sanity': t.duration('sanity'), - 'time_performance': t.duration('performance'), + 'time_setup': t.duration('setup'), 'time_total': t.duration('total') } partition = check.current_partition @@ -116,47 +116,66 @@ def json(self, reframe_info=None, force=False): if check.job: entry['jobid'] = check.job.jobid - entry['nodelist'] = check.job.nodelist or [] - entry['job_stdout'] = check.stdout.evaluate() entry['job_stderr'] = check.stderr.evaluate() + entry['job_stdout'] = check.stdout.evaluate() + entry['nodelist'] = check.job.nodelist or [] - if check._build_job: - entry['build_stdout'] = check.build_stdout.evaluate() + if check.build_job: entry['build_stderr'] = check.build_stderr.evaluate() + entry['build_stdout'] = check.build_stdout.evaluate() if t.failed: - entry['result'] = 'fail' + num_failures += 1 + entry['result'] = 'failure' + entry['stagedir'] = check.stagedir + entry['fail_phase'] = t.failed_stage if t.exc_info is not None: - entry['failing_reason'] = format_exception( - *t.exc_info) - entry['failing_phase'] = t.failed_stage - entry['stagedir'] = check.stagedir + entry['fail_reason'] = format_exception(*t.exc_info) else: entry['result'] = 'success' entry['outputdir'] = check.outputdir - tests.append(entry) - - self._records['run_info'].append({'runid': run_no, 'tests': tests}) - - if reframe_info: - self._records['reframe_info'] = reframe_info - - return self._records + if check.perf_patterns: + # Record performance variables + entry['perfvars'] = [] + for key, ref in check.perfvalues.items(): + var = key.split(':')[-1] + val, ref, lower, upper, unit = ref + entry['perfvars'].append({ + 'name': var, + 'reference': ref, + 'thres_lower': lower, + 'thres_upper': upper, + 'unit': unit, + 'value': val + }) + + testcases.append(entry) + + self._run_data.append({ + 'num_cases': len(run), + 'num_failures': num_failures, + 'runid': runid, + 'testcases': testcases + }) + + return self._run_data def failure_report(self): line_width = 78 report = [line_width * '='] report.append('SUMMARY OF FAILURES') - last_run = rt.runtime().current_run - for r in self.json()['run_info'][last_run]['tests']: + run_report = self.json()[-1] + last_run = run_report['runid'] + for r in run_report['testcases']: if r['result'] == 'success': continue - retry_info = (f'(for the last of {last_run} retries)' - if last_run > 0 else '') + retry_info = ( + f'(for the last of {last_run} retries)' if last_run > 0 else '' + ) report.append(line_width * '-') - report.append(f"FAILURE INFO for {r['testname']} {retry_info}") + report.append(f"FAILURE INFO for {r['name']} {retry_info}") report.append(f" * Test Description: {r['description']}") report.append(f" * System partition: {r['system']}") report.append(f" * Environment: {r['environment']}") @@ -167,13 +186,13 @@ def failure_report(self): jobid = r['jobid'] report.append(f" * Job type: {job_type} (id={r['jobid']})") report.append(f" * Maintainers: {r['maintainers']}") - report.append(f" * Failing phase: {r['failing_phase']}") - report.append(f" * Rerun with '-n {r['testname']}" + report.append(f" * Failing phase: {r['fail_phase']}") + report.append(f" * Rerun with '-n {r['name']}" f" -p {r['environment']} --system {r['system']}'") - report.append(f" * Reason: {r['failing_reason']}") - if r['failing_phase'] == 'sanity': + report.append(f" * Reason: {r['fail_reason']}") + if r['fail_phase'] == 'sanity': report.append('Sanity check failure') - elif r['failing_phase'] == 'performance': + elif r['fail_phase'] == 'performance': report.append('Performance check failure') else: # This shouldn't happen... @@ -227,6 +246,8 @@ def failure_stats(self): return '' def performance_report(self): + # FIXME: Adapt this function to use the JSON report + line_width = 78 report_start = line_width * '=' report_title = 'PERFORMANCE REPORT' diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index e9230f082e..b2724e4891 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -163,7 +163,10 @@ "descr": {"type": "string"}, "scheduler": { "type": "string", - "enum": ["local", "pbs", "slurm", "squeue", "torque"] + "enum": [ + "local", "pbs", "slurm", + "squeue", "torque" + ] }, "launcher": { "type": "string", @@ -361,6 +364,7 @@ }, "non_default_craype": {"type": "boolean"}, "purge_environment": {"type": "boolean"}, + "report_file": {"type": "string"}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, @@ -403,6 +407,7 @@ "general/module_mappings": [], "general/non_default_craype": false, "general/purge_environment": false, + "general/report_file": "${HOME}/.reframe/reports/run-report-{sessionid}.json", "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", diff --git a/reframe/schemas/runreport.json b/reframe/schemas/runreport.json index fff4ceed7a..b26f15ec56 100644 --- a/reframe/schemas/runreport.json +++ b/reframe/schemas/runreport.json @@ -4,93 +4,112 @@ "title": "Validation schema for ReFrame's run report", "type": "object", "properties": { - "reframe_info": { + "session_info": { "type": "object", "properties": { - "version": {"type": "string"}, - "command": {"type": "string"}, + "cmdline": {"type": "string"}, + "config_file": {"type": ["string", "null"]}, + "data_version": {"type": "string"}, + "hostname": {"type": "string"}, + "num_cases": {"type": "number"}, + "num_failures": {"type": "number"}, + "prefix_output": {"type": "string"}, + "prefix_stage": {"type": "string"}, + "time_elapsed": {"type": "number"}, + "time_end": {"type": "string"}, + "time_start": {"type": "string"}, "user": {"type": "string"}, - "host": {"type": "string"}, - "check_search_path": {"type": "string"}, - "working_directory": {"type": "string"}, - "recursive_search_path": {"type": "boolean"}, - "settings_file": {"type": "string"}, - "stage_prefix": {"type": "string"}, - "output_prefix": {"type": "string"}, - "start_time": {"type": "string"}, - "end_time": {"type": "string"} + "version": {"type": "string"}, + "workdir": {"type": "string"} }, - "additionalProperties": false, - "required": ["check_search_path", "command", "end_time", "host", - "output_prefix", "recursive_search_path", - "stage_prefix", "start_time", "user", "version", - "working_directory"] + "required": ["data_version"] }, - "run_info": { + "runs": { "type": "array", "items": { "type": "object", "properties": { + "num_cases": {"type": "number"}, + "num_failures": {"type": "number"}, "runid": {"type": "number"}, - "tests": { + "testcases": { "type": "array", "items": { "type": "object", "properties": { - "testname": {"type": "string"}, + "build_stderr": {"type": ["string", "null"]}, + "build_stdout": {"type": ["string", "null"]}, "description": {"type": "string"}, - "system": {"type": "string"}, "environment": {"type": ["string", "null"]}, - "stagedir": {"type": ["string", "null"]}, - "outputdir": {"type": ["string", "null"]}, + "fail_phase": {"type": ["string", "null"]}, + "fail_reason": {"type": ["string", "null"]}, + "jobid": {"type": ["number", "null"]}, + "job_stderr": {"type": ["string", "null"]}, + "job_stdout": {"type": ["string", "null"]}, + "name": {"type": "string"}, + "maintainers": { + "type": "array", + "items": {"type": "string"} + }, "nodelist": { "type": "array", "items": {"type": "string"} }, - "scheduler": {"type": ["string", "null"]}, - "jobid": {"type": ["number", "null"]}, + "outputdir": {"type": ["string", "null"]}, + "perfvars": { + "type": ["array", "null"], + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "reference": { + "type": ["number", "null"] + }, + "thres_lower": { + "type": ["number", "null"] + }, + "thres_upper": { + "type": ["number", "null"] + }, + "unit": {"type": ["string", "null"]}, + "value": {"type": "number"} + }, + "required": [ + "name", "reference", + "thres_lower", "thres_upper", + "unit", "value" + ] + } + }, "result": { "type": "string", - "enum": ["success", "fail"] - }, - "failing_phase": {"type": ["string", "null"]}, - "failing_reason": {"type": ["string", "null"]}, - "build_stdout": {"type": ["string", "null"]}, - "build_stderr": {"type": ["string", "null"]}, - "job_stdout": {"type": ["string", "null"]}, - "job_stderr": {"type": ["string", "null"]}, - "maintainers": { - "type": "array", - "items": {"type": ["string"]} + "enum": ["success", "failure"] }, + "scheduler": {"type": ["string", "null"]}, + "stagedir": {"type": ["string", "null"]}, + "system": {"type": "string"}, "tags": { "type": "array", "items": {"type": "string"} }, - "runid": {"type": "number"}, - "time_setup": {"type": ["number", "null"]}, "time_compile": {"type": ["number", "null"]}, + "time_performance": {"type": ["number", "null"]}, "time_run": {"type": ["number", "null"]}, "time_sanity": {"type": ["number", "null"]}, - "time_performance": {"type": ["number", "null"]}, + "time_setup": {"type": ["number", "null"]}, "time_total": {"type": ["number", "null"]} }, - "additionalProperties": false, - "required": ["build_stderr", "build_stdout", "description", - "environment", "failing_phase", "failing_reason", - "job_stderr", "job_stdout", "jobid", "maintainers", - "nodelist", "outputdir", "result", "scheduler", - "stagedir", "system", "tags", "testname", - "time_compile", "time_performance", "time_run", - "time_sanity", "time_setup", "time_total"] + "required": [ + "environment", "name", "result", "system" + ] }, - "additionalProperties": false, - "required": ["runid", "tests"] + "required": [ + "num_cases", "num_failures", "runid", "testcases" + ] } } } } }, - "additionalProperties": false, - "required": ["reframe_info", "run_info"] + "required": ["runs"] } diff --git a/reframe/utility/__init__.py b/reframe/utility/__init__.py index cc93ad53ae..b00ec6956b 100644 --- a/reframe/utility/__init__.py +++ b/reframe/utility/__init__.py @@ -17,23 +17,6 @@ from collections import UserDict -def get_next_runreport_index(runreport_dir): - runreports = os.listdir(runreport_dir) - pattern = r'runreport-\d+.json' - filenames = filter(re.compile(pattern).match, runreports) - - if not filenames: - return 1 - - idx_list = sorted([int(f.split('.')[0].split('-')[1]) - for f in filenames]) - for i, idx in enumerate(idx_list): - if idx != i: - return i - - return idx + 1 - - def seconds_to_hms(seconds): '''Convert time in seconds to a tuple of ``(hour, minutes, seconds)``.''' diff --git a/unittests/test_cli.py b/unittests/test_cli.py index a5cf59b1ea..d36a1fe964 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -108,6 +108,9 @@ def _run_reframe(system='generic:default', if more_options: argv += more_options + # Always pass the --report-file option, because we don't want to + # pollute the user's home directory + argv += [f'--report-file={tmp_path / "report.json"}'] return run_command_inline(argv, cli.main) return _run_reframe diff --git a/unittests/test_policies.py b/unittests/test_policies.py index c4aab2f6dd..6ac2afc890 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -3,13 +3,13 @@ # # SPDX-License-Identifier: BSD-3-Clause -import datetime import json import jsonschema import os import pytest import socket import sys +import time import reframe import reframe.core.runtime as rt @@ -111,42 +111,48 @@ def num_failures_stage(runner, stage): return len([t for t in stats.failures() if t.failed_stage == stage]) -def validate_report(runreport): - schema_filename = os.path.join('reframe/schemas/runreport.json') +def _validate_runreport(report): + schema_filename = 'reframe/schemas/runreport.json' with open(schema_filename) as fp: schema = json.loads(fp.read()) - jsonschema.validate(runreport, schema) + jsonschema.validate(report, schema) def test_runall(make_runner, make_cases, common_exec_ctx): - reframe_info = { - 'version': os_ext.reframe_version(), - 'command': repr(' '.join(sys.argv)), - 'user': f"{os_ext.osuser() or ''}", - 'host': socket.gethostname(), - 'working_directory': repr(os.getcwd()), - 'check_search_path': 'unittests/resources/checks', - 'recursive_search_path': True, - 'settings_file': fixtures.TEST_CONFIG_FILE, - 'stage_prefix': repr(rt.runtime().stage_prefix), - 'output_prefix': repr(rt.runtime().output_prefix), - } runner = make_runner() - reframe_info['start_time'] = datetime.datetime.today().strftime('%c %Z') + time_start = time.time() runner.runall(make_cases()) - reframe_info['end_time'] = datetime.datetime.today().strftime('%c %Z') - stats = runner.stats - runreport = runner.stats.json(reframe_info, force=True) - validate_report(runreport) - assert 8 == stats.num_cases() + time_end = time.time() + assert 8 == runner.stats.num_cases() assert_runall(runner) - assert 5 == len(stats.failures()) + assert 5 == len(runner.stats.failures()) assert 2 == num_failures_stage(runner, 'setup') assert 1 == num_failures_stage(runner, 'sanity') assert 1 == num_failures_stage(runner, 'performance') assert 1 == num_failures_stage(runner, 'cleanup') + # Create a run report and validate it + run_stats = runner.stats.json() + report = { + 'session_info': { + 'cmdline': ' '.join(sys.argv), + 'config_file': rt.runtime().site_config.filename, + 'data_version': '1.0', + 'hostname': socket.gethostname(), + 'prefix_output': rt.runtime().output_prefix, + 'prefix_stage': rt.runtime().stage_prefix, + 'user': os_ext.osuser(), + 'version': os_ext.reframe_version(), + 'workdir': os.getcwd(), + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'] + + }, + 'runs': run_stats + } + _validate_runreport(report) + def test_runall_skip_system_check(make_runner, make_cases, common_exec_ctx): runner = make_runner() From 3f006e8ed5254d400b55eb4a80d111bc7333451b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 24 Jul 2020 00:18:42 +0200 Subject: [PATCH 12/15] Fix PEP8 issue --- reframe/frontend/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index bbee0720f9..91f64c3fb6 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -71,7 +71,7 @@ def list_checks(checks, printer, detailed=False): def generate_report_filename(filepatt): - if not '{sessionid}' in filepatt: + if '{sessionid}' not in filepatt: return filepatt search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') From 272e1aae8dd535cbcb26783de121f9de3cc08971 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 24 Jul 2020 00:27:10 +0200 Subject: [PATCH 13/15] Minor fixes --- reframe/frontend/cli.py | 2 +- unittests/test_policies.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 91f64c3fb6..5c60ad38c5 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -765,7 +765,7 @@ def print_infoline(param, value): ) os.makedirs(os.path.dirname(report_file), exist_ok=True) - # Build final report JSON + # Build final JSON report run_stats = runner.stats.json() session_info.update({ 'num_cases': run_stats[0]['num_cases'], diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 6ac2afc890..1df2975fa8 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -140,14 +140,20 @@ def test_runall(make_runner, make_cases, common_exec_ctx): 'config_file': rt.runtime().site_config.filename, 'data_version': '1.0', 'hostname': socket.gethostname(), + 'num_cases': run_stats[0]['num_cases'], + 'num_failures': run_stats[-1]['num_failures'], 'prefix_output': rt.runtime().output_prefix, 'prefix_stage': rt.runtime().stage_prefix, + 'time_elapsed': time_end - time_start, + 'time_end': time.strftime( + '%FT%T%z', time.localtime(time_end), + ), + 'time_start': time.strftime( + '%FT%T%z', time.localtime(time_start), + ), 'user': os_ext.osuser(), 'version': os_ext.reframe_version(), - 'workdir': os.getcwd(), - 'num_cases': run_stats[0]['num_cases'], - 'num_failures': run_stats[-1]['num_failures'] - + 'workdir': os.getcwd() }, 'runs': run_stats } From c5116f08495ed85c93463de8706af13b504db724 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 24 Jul 2020 11:30:37 +0200 Subject: [PATCH 14/15] Document JSON report - Also a minor fix in the report name generation --- docs/config_reference.rst | 10 +++++ docs/manpage.rst | 27 ++++++++++++ docs/tutorial_basics.rst | 91 ++++++++++++++++++++++++++++++++++----- reframe/frontend/cli.py | 9 ++-- 4 files changed, 123 insertions(+), 14 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 6b34274f5e..9b46426d0f 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1111,6 +1111,16 @@ General Configuration Purge any loaded environment modules before running any tests. +.. js:attribute:: .general[].report_file + + :required: No + :default: ``"${HOME}/.reframe/reports/run-report-{sessionid}.json"`` + + The file where ReFrame will store its report. + + .. versionadded:: 3.1 + + .. js:attribute:: .general[].save_log_files :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index 520a6b9599..0722a51a2b 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -225,6 +225,8 @@ Options controlling ReFrame output Normally, if the stage directory of a test exists, ReFrame will remove it and recreate it. This option disables this behavior. + This option can also be set using the :envvar:`RFM_CLEAN_STAGEDIR` environment variable or the :js:attr:`clean_stagedir` general configuration parameter. + .. versionadded:: 3.1 .. option:: --save-log-files @@ -236,6 +238,16 @@ Options controlling ReFrame output This option can also be set using the :envvar:`RFM_SAVE_LOG_FILES` environment variable or the :js:attr:`save_log_files` general configuration parameter. +.. option:: --report-file=FILE + + The file where ReFrame will store its report. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. + + This option can also be set using the :envvar:`RFM_REPORT_FILE` environment variable or the :js:attr:`report_file` general configuration parameter. + + .. versionadded:: 3.1 + + ------------------------------------- Options controlling ReFrame execution ------------------------------------- @@ -790,6 +802,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_REPORT_FILE + + The file where ReFrame will store its report. + + .. versionadded:: 3.1 + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--report-file` + Associated configuration parameter :js:attr:`report_file` general configuration parameter + ================================== ================== + + .. envvar:: RFM_SAVE_LOG_FILES Save ReFrame log files in the output directory before exiting. diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index b57a740c3b..0371483666 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -90,28 +90,28 @@ Now it's time to run our first test: .. code-block:: none [ReFrame Setup] - version: 3.1-dev0 (rev: 986c3505) - command: './bin/reframe -c tutorials/basics/hello/hello1.py -r' - launched by: user@tresa.local - working directory: '/Users/user/reframe' + version: 3.1-dev2 (rev: 272e1aae) + command: ./bin/reframe -c tutorials/basics/hello/hello1.py -r + launched by: user@dhcp-133-44.cscs.ch + working directory: '/Users/user/Repositories/reframe' settings file: '' - check search path: '/Users/user/reframe/tutorials/basics/hello/hello1.py' - stage directory: '/Users/user/reframe/stage' - output directory: '/Users/user/reframe/output' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/hello/hello1.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' [==========] Running 1 check(s) - [==========] Started on Sat Jun 20 09:44:52 2020 + [==========] Started on Fri Jul 24 11:05:46 2020 [----------] started processing HelloTest (HelloTest) [ RUN ] HelloTest on generic:default using builtin [----------] finished processing HelloTest (HelloTest) [----------] waiting for spawned checks to finish - [ OK ] (1/1) HelloTest on generic:default using builtin [compile: 0.735s run: 0.505s total: 1.272s] + [ OK ] (1/1) HelloTest on generic:default using builtin [compile: 0.378s run: 0.299s total: 0.712s] [----------] all spawned checks have finished [ PASSED ] Ran 1 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Sat Jun 20 09:44:53 2020 + [==========] Finished on Fri Jul 24 11:05:47 2020 Perfect! We have verified that we have a functioning C compiler in our system. @@ -121,7 +121,7 @@ On successful outcome of the test, the stage directory is removed by default, bu The prefixes of these directories are printed in the first section of the output. Let's inspect what files ReFrame produced for this test: -.. code-block:: bash +.. code-block:: console ls output/generic/default/builtin/HelloTest/ @@ -133,6 +133,75 @@ Let's inspect what files ReFrame produced for this test: ReFrame stores in the output directory of the test the build and run scripts it generated for building and running the code along with their standard output and error. All these files are prefixed with ``rfm_``. +ReFrame also generates a detailed JSON report for the whole regression testing session. +By default, this is stored inside the ``${HOME}/.reframe/reports`` directory and a new report file is generated every time ReFrame is run, but you can control this through the :option:`--report-file` command-line option. + +Here are the contents of the report file for our first ReFrame run: + + +.. code-block:: console + + cat ~/.reframe/reports/run-report-0.json + +.. code-block:: javascript + + { + "session_info": { + "cmdline": "./bin/reframe -c tutorials/basics/hello/hello1.py -r", + "config_file": "", + "data_version": "1.0", + "hostname": "dhcp-133-44.cscs.ch", + "prefix_output": "/Users/user/Repositories/reframe/output", + "prefix_stage": "/Users/user/Repositories/reframe/stage", + "user": "user", + "version": "3.1-dev2 (rev: 272e1aae)", + "workdir": "/Users/user/Repositories/reframe", + "time_start": "2020-07-24T11:05:46+0200", + "time_end": "2020-07-24T11:05:47+0200", + "time_elapsed": 0.7293069362640381, + "num_cases": 1, + "num_failures": 0 + }, + "runs": [ + { + "num_cases": 1, + "num_failures": 0, + "runid": 0, + "testcases": [ + { + "build_stderr": "rfm_HelloTest_build.err", + "build_stdout": "rfm_HelloTest_build.out", + "description": "HelloTest", + "environment": "builtin", + "fail_reason": null, + "fail_phase": null, + "jobid": 85063, + "job_stderr": "rfm_HelloTest_job.err", + "job_stdout": "rfm_HelloTest_job.out", + "name": "HelloTest", + "maintainers": [], + "nodelist": [ + "dhcp-133-44.cscs.ch" + ], + "outputdir": "/Users/user/Repositories/reframe/output/generic/default/builtin/HelloTest", + "perfvars": null, + "result": "success", + "stagedir": null, + "scheduler": "local", + "system": "generic:default", + "tags": [], + "time_compile": 0.3776402473449707, + "time_performance": 4.506111145019531e-05, + "time_run": 0.2992382049560547, + "time_sanity": 0.0005609989166259766, + "time_setup": 0.0031709671020507812, + "time_total": 0.7213571071624756 + } + ] + } + ] + } + More of "Hello, World!" ----------------------- diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5c60ad38c5..074568cd54 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -76,7 +76,8 @@ def generate_report_filename(filepatt): search_patt = os.path.basename(filepatt).replace('{sessionid}', r'(\d+)') new_id = -1 - for filename in os.listdir(os.path.dirname(filepatt)): + basedir = os.path.dirname(filepatt) or '.' + for filename in os.listdir(basedir): match = re.match(search_patt, filename) if match: found_id = int(match.group(1)) @@ -547,7 +548,7 @@ def print_infoline(param, value): # Print command line printer.info(f"[ReFrame Setup]") print_infoline('version', session_info['version']) - print_infoline('command', session_info['cmdline']) + print_infoline('command', repr(session_info['cmdline'])) print_infoline( f"launched by", f"{session_info['user'] or ''}@{session_info['hostname']}" @@ -763,7 +764,9 @@ def print_infoline(param, value): report_file = os.path.normpath( os_ext.expandvars(rt.get_option('general/0/report_file')) ) - os.makedirs(os.path.dirname(report_file), exist_ok=True) + basedir = os.path.dirname(report_file) + if basedir: + os.makedirs(basedir, exist_ok=True) # Build final JSON report run_stats = runner.stats.json() From 65b1333de4ce33dcb0cde6530b65554b16d3f0e1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Fri, 24 Jul 2020 11:55:04 +0200 Subject: [PATCH 15/15] Modify unit tests to increase coverage --- unittests/test_cli.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/unittests/test_cli.py b/unittests/test_cli.py index d36a1fe964..27e5e10fb4 100644 --- a/unittests/test_cli.py +++ b/unittests/test_cli.py @@ -75,7 +75,10 @@ def _run_reframe(system='generic:default', perflogdir=str(perflogdir)): import reframe.frontend.cli as cli - argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor'] + # We always pass the --report-file option, because we don't want to + # pollute the user's home directory + argv = ['./bin/reframe', '--prefix', str(tmp_path), '--nocolor', + f'--report-file={tmp_path / "report.json"}'] if mode: argv += ['--mode', mode] @@ -108,9 +111,6 @@ def _run_reframe(system='generic:default', if more_options: argv += more_options - # Always pass the --report-file option, because we don't want to - # pollute the user's home directory - argv += [f'--report-file={tmp_path / "report.json"}'] return run_command_inline(argv, cli.main) return _run_reframe @@ -148,7 +148,19 @@ def test_check_success(run_reframe, tmp_path, logfile): assert 'PASSED' in stdout assert 'FAILED' not in stdout assert returncode == 0 - os.path.exists(tmp_path / 'output' / logfile) + assert os.path.exists(tmp_path / 'output' / logfile) + assert os.path.exists(tmp_path / 'report.json') + + +def test_report_file_with_sessionid(run_reframe, tmp_path): + returncode, stdout, _ = run_reframe( + more_options=[ + f'--save-log-files', + f'--report-file={tmp_path / "rfm-report-{sessionid}.json"}' + ] + ) + assert returncode == 0 + assert os.path.exists(tmp_path / 'rfm-report-0.json') def test_check_submit_success(run_reframe, remote_exec_ctx):