diff --git a/docs/config_reference.rst b/docs/config_reference.rst index c9ecaf8c08..31fbf54a0e 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1224,6 +1224,17 @@ General Configuration Default value has changed to avoid generating a report file per session. +.. js:attribute:: .general[].report_junit + + :required: No + :default: ``null`` + + The file where ReFrame will store its report in JUnit format. + The report adheres to the XSD schema `here `__. + + .. versionadded:: 3.6.0 + + .. js:attribute:: .general[].resolve_module_conflicts :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index 0564c4d3f0..bc95f0307d 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -274,6 +274,16 @@ Options controlling ReFrame output .. versionadded:: 3.1 +.. option:: --report-junit=FILE + + Instruct ReFrame to generate a JUnit XML report in ``FILE``. + The generated report adheres to the XSD schema `here `__ and it takes into account only the first run, ignoring retries of failed tests. + + This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter. + + .. versionadded:: 3.6.0 + + ------------------------------------- Options controlling ReFrame execution ------------------------------------- @@ -859,6 +869,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_REPORT_JUNIT + + The file where ReFrame will generate a JUnit XML report. + + .. versionadded:: 3.6.0 + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--report-junit` + Associated configuration parameter :js:attr:`report_junit` general configuration parameter + ================================== ================== + + .. envvar:: RFM_RESOLVE_MODULE_CONFLICTS Resolve module conflicts automatically. diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 69c9952669..2c22342424 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -215,6 +215,12 @@ def main(): envvar='RFM_REPORT_FILE', configvar='general/report_file' ) + output_options.add_argument( + '--report-junit', action='store', metavar='FILE', + help="Store a JUnit report in FILE", + envvar='RFM_REPORT_JUNIT', + configvar='general/report_junit' + ) # Check discovery options locate_options.add_argument( @@ -1045,6 +1051,21 @@ def module_unuse(*paths): f'failed to generate report in {report_file!r}: {e}' ) + # Generate the junit xml report for this session + junit_report_file = rt.get_option('general/0/report_junit') + if junit_report_file: + # Expand variables in filename + junit_report_file = osext.expandvars(junit_report_file) + junit_xml = runreport.junit_xml_report(json_report) + try: + with open(junit_report_file, 'w') as fp: + runreport.junit_dump(junit_xml, fp) + except OSError as e: + printer.warning( + f'failed to generate report in {junit_report_file!r}: ' + f'{e}' + ) + if not success: sys.exit(1) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 1cedb843ec..3df8eaf675 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -3,8 +3,10 @@ # # SPDX-License-Identifier: BSD-3-Clause +import decimal import json import jsonschema +import lxml.etree as etree import os import re @@ -13,7 +15,6 @@ import reframe.utility.jsonext as jsonext import reframe.utility.versioning as versioning - DATA_VERSION = '1.3.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') @@ -152,3 +153,62 @@ def load_report(filename): ) return _RunReport(report) + + +def junit_xml_report(json_report): + '''Generate a JUnit report from a standard ReFrame JSON report.''' + + xml_testsuites = etree.Element('testsuites') + xml_testsuite = etree.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'errors': '0', + 'failures': str(json_report['session_info']['num_failures']), + 'hostname': json_report['session_info']['hostname'], + 'id': '0', + 'name': 'reframe', + 'package': 'reframe', + 'tests': str(json_report['session_info']['num_cases']), + 'time': str(json_report['session_info']['time_elapsed']), + + # XSD schema does not like the timezone format, so we remove it + 'timestamp': json_report['session_info']['time_start'][:-5], + } + ) + testsuite_properties = etree.SubElement(xml_testsuite, 'properties') + for testid in range(len(json_report['runs'][0]['testcases'])): + tid = json_report['runs'][0]['testcases'][testid] + casename = ( + f"{tid['name']}[{tid['system']}, {tid['environment']}]" + ) + testcase = etree.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tid['filename'], + 'name': casename, + + # XSD schema does not like the exponential format and since we + # do not want to impose a fixed width, we pass it to `Decimal` + # to format it automatically. + 'time': str(decimal.Decimal(tid['time_total'])), + } + ) + if tid['result'] == 'failure': + testcase_msg = etree.SubElement( + testcase, 'failure', attrib={'type': 'failure', + 'message': tid['fail_phase']} + ) + testcase_msg.text = f"{tid['fail_phase']}: {tid['fail_reason']}" + + testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out') + testsuite_stdout.text = '' + testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err') + testsuite_stderr.text = '' + return xml_testsuites + + +def junit_dump(xml, fp): + fp.write( + etree.tostring(xml, encoding='utf8', pretty_print=True, + method='xml', xml_declaration=True).decode() + ) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 963676d519..5ee6a757d5 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -5,6 +5,7 @@ import inspect import traceback + import reframe.core.runtime as rt import reframe.core.exceptions as errors import reframe.utility as util @@ -266,8 +267,8 @@ def print_failure_stats(self, printer): stats_header = row_format.format('Phase', '#', 'Failing test cases') num_tests = len(self.tasks(current_run)) num_failures = 0 - for l in failures.values(): - num_failures += len(l) + for fl in failures.values(): + num_failures += len(fl) stats_body = [''] stats_body.append(f'Total number of test cases: {num_tests}') diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 391b91187a..1867c8dafc 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -448,6 +448,7 @@ "non_default_craype": {"type": "boolean"}, "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, + "report_junit": {"type": ["string", "null"]}, "resolve_module_conflicts": {"type": "boolean"}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, @@ -487,6 +488,7 @@ "general/non_default_craype": false, "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report.json", + "general/report_junit": null, "general/resolve_module_conflicts": true, "general/save_log_files": false, "general/target_systems": ["*"], diff --git a/reframe/schemas/junit.xsd b/reframe/schemas/junit.xsd new file mode 100644 index 0000000000..84b0f157b1 --- /dev/null +++ b/reframe/schemas/junit.xsd @@ -0,0 +1,212 @@ + + + + JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks +Copyright © 2011, Windy Road Technology Pty. Limited +The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ +Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). + + + + + + + + + + Contains an aggregation of testsuite results + + + + + + + + + + Derived from testsuite/@name in the non-aggregated documents + + + + + Starts at '0' for the first testsuite and is incremented by 1 for each following testsuite + + + + + + + + + + + + Contains the results of exexuting a testsuite + + + + + Properties (e.g., environment settings) set during test execution + + + + + + + + + + + + + + + + + + + + + + + + + Indicates that the test errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. Contains as a text node relevant data for the error, e.g., a stack trace + + + + + + + The error message. e.g., if a java exception is thrown, the return value of getMessage() + + + + + The type of error that occured. e.g., if a java execption is thrown the full class name of the exception. + + + + + + + + + Indicates that the test failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals. Contains as a text node relevant data for the failure, e.g., a stack trace + + + + + + + The message specified in the assert + + + + + The type of the assert. + + + + + + + + + + Name of the test method + + + + + Full class name for the class the test method is in. + + + + + Time taken (in seconds) to execute the test + + + + + + + Data that was written to standard out while the test was executed + + + + + + + + + + Data that was written to standard error while the test was executed + + + + + + + + + + + Full class name of the test for non-aggregated testsuite documents. Class name without the package for aggregated testsuites documents + + + + + + + + + + when the test was executed. Timezone may not be specified. + + + + + Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. + + + + + + + + + + The total number of tests in the suite + + + + + The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals + + + + + The total number of tests in the suite that errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. + + + + + The total number of ignored or skipped tests in the suite. + + + + + Time taken (in seconds) to execute the tests in the suite + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 8a717f178f..5a03c53e0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ argcomplete==1.12.3 coverage==5.5 importlib_metadata==4.0.1; python_version < '3.8' jsonschema==3.2.0 +lxml==4.6.3 pytest==6.2.3 pytest-forked==1.3.0 pytest-parallel==0.1.0 diff --git a/setup.py b/setup.py index 2013fc593a..b698fc23a3 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ), package_data={'reframe': ['schemas/*']}, include_package_data=True, - install_requires=['argcomplete', 'jsonschema', 'PyYAML', 'semver'], + install_requires=['argcomplete', 'jsonschema', 'lxml', 'PyYAML', 'semver'], python_requires='>=3.6', scripts=['bin/reframe'], classifiers=( diff --git a/unittests/test_policies.py b/unittests/test_policies.py index d03549a9f6..52f462e85e 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -21,6 +21,7 @@ import reframe.utility.osext as osext import unittests.utility as test_util +from lxml import etree from reframe.core.exceptions import (AbortTaskError, FailureLimitError, ReframeError, @@ -178,6 +179,16 @@ def _validate_runreport(report): jsonschema.validate(json.loads(report), schema) +def _validate_junit_report(report): + # Cloned from + # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd + schema_file = 'reframe/schemas/junit.xsd' + with open(schema_file, encoding='utf-8') as fp: + schema = etree.XMLSchema(etree.parse(fp)) + + schema.assert_(report) + + def _generate_runreport(run_stats, time_start, time_end): return { 'session_info': { @@ -226,6 +237,10 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): with open(report_file, 'w') as fp: jsonext.dump(report, fp) + # Validate the junit report + xml_report = runreport.junit_xml_report(report) + _validate_junit_report(xml_report) + # Read and validate the report using the runreport module runreport.load_report(report_file)