From 566362839d590d6c144f6dd51a474fcb4dab1625 Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 8 Apr 2021 17:23:02 +0200 Subject: [PATCH 01/26] xml --- reframe/frontend/cli.py | 12 +++++++++++ reframe/frontend/statistics.py | 38 ++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index e107ec5c67..71d445949f 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -26,6 +26,7 @@ import reframe.frontend.filters as filters import reframe.frontend.runreport as runreport import reframe.utility.jsonext as jsonext +# import reframe.utility.xmlext as xmlext import reframe.utility.osext as osext @@ -1021,6 +1022,17 @@ def module_unuse(*paths): f'failed to generate report in {report_file!r}: {e}' ) + xml_data = runner.stats.junit(json_report).decode() + xml_report_file = f'{report_file}.xml' + try: + with open(xml_report_file, 'w') as fp: + fp.write(str(xml_data)) + fp.write('\n') + except OSError as e: + printer.warning( + f'failed to generate report in {xml_report_file!r}: {e}' + ) + if not success: sys.exit(1) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 16db93d2d5..d54f80f2c5 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -5,6 +5,7 @@ import inspect import traceback +import xml.etree.ElementTree as ET import reframe.core.runtime as rt import reframe.core.exceptions as errors @@ -197,6 +198,39 @@ def json(self, force=False): return self._run_data + def junit(self, json_report, force=False): + # https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd + xml_testsuites = ET.Element('testsuites') + xml_testsuite = ET.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'name': 'rfm', + 'errors': '0', + 'failures': str(json_report['session_info']['num_failures']), + 'tests': str(json_report['session_info']['num_cases']), + 'time': str(json_report['session_info']['time_elapsed']), + # 'hostname': 'dom', + } + ) + + for testid in range(len(json_report['runs'][0]['testcases'])): + tid = json_report['runs'][0]['testcases'][testid] + testcase = ET.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tid['filename'], + 'name': tid['name'], + 'time': str(tid['time_total']), + } + ) + if not tid['result'] == 'success': + testcase_msg = ET.SubElement( + testcase, 'failure', attrib={'message': tid['fail_phase']} + ) + testcase_msg.text = tid['fail_reason'] + + return ET.tostring(xml_testsuites, encoding='utf8', method='xml') + def print_failure_report(self, printer): line_width = 78 printer.info(line_width * '=') @@ -264,8 +298,8 @@ def print_failure_stats(self, printer): stats_header = row_format.format('Phase', '#', 'Failing test cases') num_tests = len(self.tasks(current_run)) num_failures = 0 - for l in failures.values(): - num_failures += len(l) + for ll in failures.values(): + num_failures += len(ll) stats_body = [''] stats_body.append(f'Total number of test cases: {num_tests}') From 898ffcb597ce10a99a65501bdc7ed73c99ef50cc Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 9 Apr 2021 11:40:17 +0200 Subject: [PATCH 02/26] rpt --- reframe/frontend/cli.py | 3 ++- reframe/frontend/statistics.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 71d445949f..7360b2f91e 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1022,8 +1022,9 @@ def module_unuse(*paths): f'failed to generate report in {report_file!r}: {e}' ) + # Build xml report xml_data = runner.stats.junit(json_report).decode() - xml_report_file = f'{report_file}.xml' + xml_report_file = os.path.splitext(report_file)[0]+'.xml' try: with open(xml_report_file, 'w') as fp: fp.write(str(xml_data)) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index d54f80f2c5..148b3b886e 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -209,17 +209,17 @@ def junit(self, json_report, force=False): 'failures': str(json_report['session_info']['num_failures']), 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), - # 'hostname': 'dom', } ) for testid in range(len(json_report['runs'][0]['testcases'])): tid = json_report['runs'][0]['testcases'][testid] + name = f"{tid['name']} on {tid['system']} using {tid['environment']}" testcase = ET.SubElement( xml_testsuite, 'testcase', attrib={ 'classname': tid['filename'], - 'name': tid['name'], + 'name': name, 'time': str(tid['time_total']), } ) From 6020d54515a6d81b8069c894cab0331da2caa010 Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 9 Apr 2021 11:45:25 +0200 Subject: [PATCH 03/26] typo --- reframe/frontend/statistics.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 148b3b886e..c0082e91da 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -214,7 +214,9 @@ def junit(self, json_report, force=False): for testid in range(len(json_report['runs'][0]['testcases'])): tid = json_report['runs'][0]['testcases'][testid] - name = f"{tid['name']} on {tid['system']} using {tid['environment']}" + name = ( + f"{tid['name']} on {tid['system']} using {tid['environment']}" + ) testcase = ET.SubElement( xml_testsuite, 'testcase', attrib={ From 6af94ee0eaa34ad40efb5061e49bd1c071a5e1ac Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 9 Apr 2021 16:15:53 +0200 Subject: [PATCH 04/26] fix for review --- reframe/frontend/cli.py | 1 - reframe/frontend/statistics.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 7360b2f91e..4be01e37f2 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -26,7 +26,6 @@ import reframe.frontend.filters as filters import reframe.frontend.runreport as runreport import reframe.utility.jsonext as jsonext -# import reframe.utility.xmlext as xmlext import reframe.utility.osext as osext diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index c0082e91da..cd2cf12119 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -6,6 +6,7 @@ import inspect import traceback import xml.etree.ElementTree as ET + import reframe.core.runtime as rt import reframe.core.exceptions as errors From 75f1bb4856604d0cda89e4eb9db259599e44ed25 Mon Sep 17 00:00:00 2001 From: jgp Date: Mon, 12 Apr 2021 13:14:42 +0200 Subject: [PATCH 05/26] fix for https://github.com/eth-cscs/reframe/pull/1925#discussion_r611396174 --- reframe/frontend/statistics.py | 1 + 1 file changed, 1 insertion(+) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index cd2cf12119..eea4047436 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -210,6 +210,7 @@ def junit(self, json_report, force=False): 'failures': str(json_report['session_info']['num_failures']), 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), + 'hostname': json_report['session_info']['hostname'], } ) From 286cc91447a985e497ced5e53fbde52ef147e975 Mon Sep 17 00:00:00 2001 From: jgp Date: Tue, 13 Apr 2021 14:50:17 +0200 Subject: [PATCH 06/26] fix for review --- docs/config_reference.rst | 10 ++++++++++ docs/manpage.rst | 13 +++++++++++++ reframe/frontend/cli.py | 12 ++++++++++-- reframe/schemas/config.json | 2 ++ 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 2651f80481..56a19be8f7 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1224,6 +1224,16 @@ General Configuration Default value has changed to avoid generating a report file per session. +.. js:attribute:: .general[].report_junit + + :required: No + :default: ``"${HOME}/.reframe/reports/run-report.xml"`` + + The file where ReFrame will store its report in junit xml format. + + .. versionadded:: 3.6 + + .. js:attribute:: .general[].save_log_files :required: No diff --git a/docs/manpage.rst b/docs/manpage.rst index b759cb1ed0..d7698d5ed7 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -266,6 +266,19 @@ Options controlling ReFrame output .. versionadded:: 3.1 +.. option:: --report-junit=FILE + + The file where ReFrame will store its report in junit xml format. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, + in which case ReFrame will generate a new report each time it is run by + appending a counter to the report file. + + This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment + variable or the :js:attr:`report_junit` general configuration parameter. + + .. versionadded:: 3.1 + + ------------------------------------- Options controlling ReFrame execution ------------------------------------- diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 4be01e37f2..5cd5a688d7 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -207,6 +207,12 @@ def main(): envvar='RFM_REPORT_FILE', configvar='general/report_file' ) + output_options.add_argument( + '--report-junit', action='store', metavar='FILE', + help="Store XML junit run report in FILE", + envvar='RFM_REPORT_JUNIT', + configvar='general/report_junit' + ) # Check discovery options locate_options.add_argument( @@ -1021,9 +1027,11 @@ def module_unuse(*paths): f'failed to generate report in {report_file!r}: {e}' ) - # Build xml report + # Generate the junit xml report for this session + xml_report_file = os.path.normpath( + osext.expandvars(rt.get_option('general/0/report_junit')) + ) xml_data = runner.stats.junit(json_report).decode() - xml_report_file = os.path.splitext(report_file)[0]+'.xml' try: with open(xml_report_file, 'w') as fp: fp.write(str(xml_data)) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 1d5b143b1c..078deb5185 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -448,6 +448,7 @@ "non_default_craype": {"type": "boolean"}, "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, + "report_junit": {"type": "string"}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, @@ -486,6 +487,7 @@ "general/non_default_craype": false, "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report.json", + "general/report_junit": "${HOME}/.reframe/reports/run-report.xml", "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", From 6ecb6ed887d6e63267225c94b585216c3ca01cf9 Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 16 Apr 2021 12:09:36 +0200 Subject: [PATCH 07/26] Fix for review --- reframe/frontend/cli.py | 4 +- reframe/frontend/statistics.py | 77 +++++++++++++++++----------------- 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 5cd5a688d7..304ab7550b 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -34,6 +34,7 @@ from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.executors import Runner, generate_testcases +from reframe.frontend.statistics import junit def format_check(check, check_deps, detailed=False): @@ -1031,7 +1032,8 @@ def module_unuse(*paths): xml_report_file = os.path.normpath( osext.expandvars(rt.get_option('general/0/report_junit')) ) - xml_data = runner.stats.junit(json_report).decode() + xml_data = junit(json_report).decode() + # xml_data = runner.stats.junit(json_report).decode() try: with open(xml_report_file, 'w') as fp: fp.write(str(xml_data)) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index eea4047436..5ebeba421a 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -199,42 +199,6 @@ def json(self, force=False): return self._run_data - def junit(self, json_report, force=False): - # https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd - xml_testsuites = ET.Element('testsuites') - xml_testsuite = ET.SubElement( - xml_testsuites, 'testsuite', - attrib={ - 'name': 'rfm', - 'errors': '0', - 'failures': str(json_report['session_info']['num_failures']), - 'tests': str(json_report['session_info']['num_cases']), - 'time': str(json_report['session_info']['time_elapsed']), - 'hostname': json_report['session_info']['hostname'], - } - ) - - for testid in range(len(json_report['runs'][0]['testcases'])): - tid = json_report['runs'][0]['testcases'][testid] - name = ( - f"{tid['name']} on {tid['system']} using {tid['environment']}" - ) - testcase = ET.SubElement( - xml_testsuite, 'testcase', - attrib={ - 'classname': tid['filename'], - 'name': name, - 'time': str(tid['time_total']), - } - ) - if not tid['result'] == 'success': - testcase_msg = ET.SubElement( - testcase, 'failure', attrib={'message': tid['fail_phase']} - ) - testcase_msg.text = tid['fail_reason'] - - return ET.tostring(xml_testsuites, encoding='utf8', method='xml') - def print_failure_report(self, printer): line_width = 78 printer.info(line_width * '=') @@ -302,8 +266,8 @@ def print_failure_stats(self, printer): stats_header = row_format.format('Phase', '#', 'Failing test cases') num_tests = len(self.tasks(current_run)) num_failures = 0 - for ll in failures.values(): - num_failures += len(ll) + for fl in failures.values(): + num_failures += len(fl) stats_body = [''] stats_body.append(f'Total number of test cases: {num_tests}') @@ -360,3 +324,40 @@ def performance_report(self): report_end]) return '' + + +def junit(json_report): + # https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd + xml_testsuites = ET.Element('testsuites') + xml_testsuite = ET.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'name': 'rfm', + 'errors': '0', + 'failures': str(json_report['session_info']['num_failures']), + 'tests': str(json_report['session_info']['num_cases']), + 'time': str(json_report['session_info']['time_elapsed']), + 'hostname': json_report['session_info']['hostname'], + } + ) + + for testid in range(len(json_report['runs'][0]['testcases'])): + tid = json_report['runs'][0]['testcases'][testid] + name = ( + f"{tid['name']} on {tid['system']} using {tid['environment']}" + ) + testcase = ET.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tid['filename'], + 'name': name, + 'time': str(tid['time_total']), + } + ) + if not tid['result'] == 'success': + testcase_msg = ET.SubElement( + testcase, 'failure', attrib={'message': tid['fail_phase']} + ) + testcase_msg.text = tid['fail_reason'] + + return ET.tostring(xml_testsuites, encoding='utf8', method='xml') From aad88e63c459e07c3b04a3d5b1dc77a4c0bc9bbc Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 16 Apr 2021 12:37:03 +0200 Subject: [PATCH 08/26] fix for review --- reframe/frontend/cli.py | 1 - reframe/frontend/statistics.py | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 304ab7550b..3ae2171fd9 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -1033,7 +1033,6 @@ def module_unuse(*paths): osext.expandvars(rt.get_option('general/0/report_junit')) ) xml_data = junit(json_report).decode() - # xml_data = runner.stats.junit(json_report).decode() try: with open(xml_report_file, 'w') as fp: fp.write(str(xml_data)) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 5ebeba421a..a310f07921 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -343,20 +343,20 @@ def junit(json_report): for testid in range(len(json_report['runs'][0]['testcases'])): tid = json_report['runs'][0]['testcases'][testid] - name = ( - f"{tid['name']} on {tid['system']} using {tid['environment']}" + casename = ( + f"{tid['name']}[{tid['system']}, {tid['environment']}]" ) testcase = ET.SubElement( xml_testsuite, 'testcase', attrib={ 'classname': tid['filename'], - 'name': name, + 'name': casename, 'time': str(tid['time_total']), } ) - if not tid['result'] == 'success': + if tid['result'] == 'failure': testcase_msg = ET.SubElement( - testcase, 'failure', attrib={'message': tid['fail_phase']} + testcase, 'failure', attrib={'type': tid['fail_phase']} ) testcase_msg.text = tid['fail_reason'] From b6eaa50d06c3b2a5d01314229de337260a6b0c82 Mon Sep 17 00:00:00 2001 From: jgp Date: Fri, 16 Apr 2021 20:06:03 +0200 Subject: [PATCH 09/26] fix for review --- reframe/frontend/cli.py | 27 +++++++++++++---------- reframe/frontend/statistics.py | 40 +++++++++++++++++++++++++++++++++- reframe/schemas/config.json | 2 +- requirements.txt | 1 + 4 files changed, 56 insertions(+), 14 deletions(-) diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 3ae2171fd9..4a32c47f2a 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -34,7 +34,7 @@ from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.executors import Runner, generate_testcases -from reframe.frontend.statistics import junit +from reframe.frontend.statistics import junit_lxml def format_check(check, check_deps, detailed=False): @@ -1029,18 +1029,21 @@ def module_unuse(*paths): ) # Generate the junit xml report for this session - xml_report_file = os.path.normpath( - osext.expandvars(rt.get_option('general/0/report_junit')) - ) - xml_data = junit(json_report).decode() - try: - with open(xml_report_file, 'w') as fp: - fp.write(str(xml_data)) - fp.write('\n') - except OSError as e: - printer.warning( - f'failed to generate report in {xml_report_file!r}: {e}' + report_xml = 'general/0/report_junit' + if site_config.get(report_xml) or os.getenv('RFM_REPORT_JUNIT'): + xml_report_file = os.path.normpath( + osext.expandvars(rt.get_option(report_xml)) ) + xml_data = junit_lxml(json_report).decode() + try: + with open(xml_report_file, 'w') as fp: + fp.write(str(xml_data)) + fp.write('\n') + except OSError as e: + printer.warning( + f'failed to generate report in {xml_report_file!r}:' + f'{e}' + ) if not success: sys.exit(1) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index a310f07921..c80776e1a7 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -6,6 +6,7 @@ import inspect import traceback import xml.etree.ElementTree as ET +import lxml.etree as LX import reframe.core.runtime as rt import reframe.core.exceptions as errors @@ -326,7 +327,7 @@ def performance_report(self): return '' -def junit(json_report): +def junit_xml(json_report): # https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd xml_testsuites = ET.Element('testsuites') xml_testsuite = ET.SubElement( @@ -361,3 +362,40 @@ def junit(json_report): testcase_msg.text = tid['fail_reason'] return ET.tostring(xml_testsuites, encoding='utf8', method='xml') + +def junit_lxml(json_report): + # https://lxml.de/tutorial.html + xml_testsuites = LX.Element('testsuites') + xml_testsuite = LX.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'name': 'rfm', + 'errors': '0', + 'failures': str(json_report['session_info']['num_failures']), + 'tests': str(json_report['session_info']['num_cases']), + 'time': str(json_report['session_info']['time_elapsed']), + 'hostname': json_report['session_info']['hostname'], + } + ) + + for testid in range(len(json_report['runs'][0]['testcases'])): + tid = json_report['runs'][0]['testcases'][testid] + casename = ( + f"{tid['name']}[{tid['system']}, {tid['environment']}]" + ) + testcase = LX.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tid['filename'], + 'name': casename, + 'time': str(tid['time_total']), + } + ) + if tid['result'] == 'failure': + testcase_msg = LX.SubElement( + testcase, 'failure', attrib={'type': tid['fail_phase']} + ) + testcase_msg.text = tid['fail_reason'] + + return LX.tostring(xml_testsuites, encoding='utf8', pretty_print=True, + method='xml', xml_declaration=True) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 078deb5185..1e35804581 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -487,7 +487,7 @@ "general/non_default_craype": false, "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report.json", - "general/report_junit": "${HOME}/.reframe/reports/run-report.xml", + "general/report_junit": "", "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", diff --git a/requirements.txt b/requirements.txt index 76bc9b3983..b5040974ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ argcomplete==1.12.2 coverage==5.5 importlib_metadata==3.10.0; python_version < '3.8' jsonschema==3.2.0 +lxml==4.6.3 pytest==6.2.2 pytest-forked==1.3.0 pytest-parallel==0.1.0 From fea4f4846cf0938da6b3c6df1264cc8c969a6347 Mon Sep 17 00:00:00 2001 From: jgp Date: Sun, 18 Apr 2021 08:42:48 +0200 Subject: [PATCH 10/26] typo --- docs/manpage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index d7698d5ed7..6d4459bd1f 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -276,7 +276,7 @@ Options controlling ReFrame output This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter. - .. versionadded:: 3.1 + .. versionadded:: 3.6 ------------------------------------- From 57c0926ce239e9db97f1e74782aa8881bddaa2b9 Mon Sep 17 00:00:00 2001 From: jgp Date: Sun, 18 Apr 2021 14:17:59 +0200 Subject: [PATCH 11/26] unittest --- unittests/test_junit.py | 243 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 unittests/test_junit.py diff --git a/unittests/test_junit.py b/unittests/test_junit.py new file mode 100644 index 0000000000..f58d3626e9 --- /dev/null +++ b/unittests/test_junit.py @@ -0,0 +1,243 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import os +import pytest +import sys + +from lxml import etree +from lxml.doctestcompare import norm_whitespace +from reframe.frontend.statistics import junit_lxml + +# https://github.com/SuminAndrew/lxml-asserts/blob/master/LICENSE + + +def raise_exc_info(exc_info): + raise exc_info[1].with_traceback(exc_info[2]) + + +def _describe_element(elem): + return elem.getroottree().getpath(elem) + + +def _xml_compare_text(t1, t2, strip): + t1 = t1 or '' + t2 = t2 or '' + + if strip: + t1 = norm_whitespace(t1).strip() + t2 = norm_whitespace(t2).strip() + + return t1 == t2 + + +def _assert_tag_and_attributes_are_equal(xml1, xml2, can_extend=False): + if xml1.tag != xml2.tag: + raise AssertionError( + u"Tags do not match: {tag1} != {tag2}".format( + tag1=_describe_element(xml1), tag2=_describe_element(xml2) + ) + ) + + added_attributes = set(xml2.attrib).difference(xml1.attrib) + missing_attributes = set(xml1.attrib).difference(xml2.attrib) + + if missing_attributes: + raise AssertionError( + u"Second xml misses attributes: {path}/({attributes})".format( + path=_describe_element(xml2), attributes=','.join( + missing_attributes) + ) + ) + + if not can_extend and added_attributes: + raise AssertionError( + (u"Second xml has additional attributes: " + u"{path}/({attributes})".format( + path=_describe_element(xml2), + attributes=','.join(added_attributes))) + ) + + for attrib in xml1.attrib: + if not _xml_compare_text(xml1.attrib[attrib], xml2.attrib[attrib], + False): + raise AssertionError( + (u"Attribute values are not equal: {path}/{attribute}" + u"['{v1}' != '{v2}']".format( + path=_describe_element(xml1), + attribute=attrib, + v1=xml1.attrib[attrib], + v2=xml2.attrib[attrib]))) + + if not _xml_compare_text(xml1.text, xml2.text, True): + raise AssertionError( + u"Tags text differs: {path}['{t1}' != '{t2}']".format( + path=_describe_element(xml1), t1=xml1.text, t2=xml2.text + ) + ) + + if not _xml_compare_text(xml1.tail, xml2.tail, True): + raise AssertionError( + u"Tags tail differs: {path}['{t1}' != '{t2}']".format( + path=_describe_element(xml1), t1=xml1.tail, t2=xml2.tail + ) + ) + + +def _assert_xml_docs_are_equal(xml1, xml2, check_tags_order=False): + _assert_tag_and_attributes_are_equal(xml1, xml2) + + children1 = list(xml1) + children2 = list(xml2) + + if len(children1) != len(children2): + raise AssertionError( + (u"Children are not equal: " + u"{len1} children != {len2} children]".format( + path=_describe_element(xml1), len1=len(children1), + len2=len(children2)))) + + raise AssertionError( + (u"Children are not equal: " + u"{path}[{len1} children != {len2} children]".format( + path=_describe_element(xml1), len1=len(children1), + len2=len(children2)))) + + if check_tags_order: + for c1, c2 in zip(children1, children2): + _assert_xml_docs_are_equal(c1, c2, True) + + else: + children1 = set(children1) + children2 = set(children2) + + for c1 in children1: + c1_match = None + + for c2 in children2: + try: + _assert_xml_docs_are_equal(c1, c2, False) + except AssertionError: + pass + else: + c1_match = c2 + break + + if c1_match is None: + raise AssertionError( + u"No equal child found in second xml: {path}".format( + path=_describe_element(c1) + ) + ) + + children2.remove(c1_match) + + +def _assert_xml_compare(cmp_func, xml1, xml2, **kwargs): + if not isinstance(xml1, etree._Element): + xml1 = etree.fromstring(xml1) + + if not isinstance(xml2, etree._Element): + xml2 = etree.fromstring(xml2) + + cmp_func(xml1, xml2, **kwargs) + + +def assert_xml_equal(first, second, check_tags_order=False): + _assert_xml_compare( + _assert_xml_docs_are_equal, first, second, + check_tags_order=check_tags_order + ) + + +def _generate_json(): + json_rpt = { + 'session_info': { + 'num_failures': '1', + 'num_cases': '2', + 'time_elapsed': '0.445', + 'hostname': 'dom101', + }, + 'runs': [ + { + 'testcases': [ + { + 'name': 'P100_Test', + 'system': 'dom:mc', + 'time_total': '0.179', + 'filename': 'test.py', + 'environment': 'PrgEnv-cray', + 'result': 'success', + }, + { + 'name': 'V100_Test', + 'system': 'dom:mc', + 'time_total': '0.266', + 'filename': 'test.py', + 'environment': 'PrgEnv-cray', + 'result': 'failure', + 'fail_phase': 'sanity', + 'fail_reason': ( + "sanity error: pattern 'x' not found in " + "'rfm_V100_Test_1_job.out'" + ), + }, + ], + } + ], + } + return json_rpt + + +def assertXmlEqual(self, first, second, check_tags_order=False, msg=None): + ''' + Assert that two xml documents are equal. + :param first: first etree object or xml string + :param second: second etree object or xml string + :param check_tags_order: if False, the order of children is ignored + :param msg: custom error message + :return: raises failureException if xml documents are not equal + ''' + if msg is None: + msg = u'XML documents are not equal' + + try: + assert_xml_equal(first, second, check_tags_order) + except AssertionError as e: + raise_exc_info( + ( + self.failureException, + self.failureException(u"{} — {}".format(msg, unicode_type(e))), + sys.exc_info()[2], + ) + ) + + +def test_xmlreport(): + # + reference_tree_str = """ + + + + + sanity error: pattern 'x' not found in + 'rfm_V100_Test_1_job.out' + + + + + """.strip() + reference_tree = etree.fromstring(reference_tree_str) + json_report = _generate_json() + rfm_tree = etree.fromstring(junit_lxml(json_report)) + # debug with: print(etree.tostring(rfm_tree).decode('utf-8')) + msg = u'XML documents are not equal' + try: + assert_xml_equal(rfm_tree, reference_tree, check_tags_order=False) + except AssertionError as e: + print(f'___{e}___') From faef8b13758512681b20e21acc1de4fa7c3d481c Mon Sep 17 00:00:00 2001 From: jgp Date: Sun, 18 Apr 2021 14:23:40 +0200 Subject: [PATCH 12/26] typo --- unittests/test_junit.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/unittests/test_junit.py b/unittests/test_junit.py index f58d3626e9..b970be75f2 100644 --- a/unittests/test_junit.py +++ b/unittests/test_junit.py @@ -14,10 +14,6 @@ # https://github.com/SuminAndrew/lxml-asserts/blob/master/LICENSE -def raise_exc_info(exc_info): - raise exc_info[1].with_traceback(exc_info[2]) - - def _describe_element(elem): return elem.getroottree().getpath(elem) From 5d9bfa52b4b1d5cc51204bebd79f047efb711fdb Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 22 Apr 2021 22:08:43 +0200 Subject: [PATCH 13/26] fix for review --- reframe/frontend/runreport.py | 34 ++++++ reframe/schemas/JUnit.xsd | 212 ++++++++++++++++++++++++++++++++++ unittests/test_policies.py | 5 +- 3 files changed, 250 insertions(+), 1 deletion(-) create mode 100644 reframe/schemas/JUnit.xsd diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 1cedb843ec..fe36ae48ac 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -13,9 +13,13 @@ import reframe.utility.jsonext as jsonext import reframe.utility.versioning as versioning +from io import BytesIO, StringIO +from lxml import etree +from reframe.frontend.statistics import junit_lxml DATA_VERSION = '1.3.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') +_JUNIT_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/JUnit.xsd') class _RunReport: @@ -152,3 +156,33 @@ def load_report(filename): ) return _RunReport(report) + + +def load_xml_report(filename): + try: + with open(filename, 'r') as fp: + json_report = json.load(fp) + except OSError as e: + raise errors.ReframeError( + f'failed to load report xml file {filename!r}') from e + except json.JSONDecodeError as e: + raise errors.ReframeError( + f'report file {filename!r} is not a valid JSON file') from e + + # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd + f = open(_JUNIT_SCHEMA, 'rb') + try: + schema = etree.XMLSchema(file=f) + finally: + f.close() + + rfm_tree = etree.fromstring(junit_lxml(json_report)) + text = etree.tostring(rfm_tree) + f = BytesIO(text) if isinstance(text, bytes) else StringIO(text) + tree_valid = etree.parse(f) + try: + schema.validate(tree_valid) + except ValidationError as e: + raise errors.ReframeError(f'invalid junit report {filename!r}') from e + + return _RunReport(json_report) diff --git a/reframe/schemas/JUnit.xsd b/reframe/schemas/JUnit.xsd new file mode 100644 index 0000000000..84b0f157b1 --- /dev/null +++ b/reframe/schemas/JUnit.xsd @@ -0,0 +1,212 @@ + + + + JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks +Copyright © 2011, Windy Road Technology Pty. Limited +The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ +Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). + + + + + + + + + + Contains an aggregation of testsuite results + + + + + + + + + + Derived from testsuite/@name in the non-aggregated documents + + + + + Starts at '0' for the first testsuite and is incremented by 1 for each following testsuite + + + + + + + + + + + + Contains the results of exexuting a testsuite + + + + + Properties (e.g., environment settings) set during test execution + + + + + + + + + + + + + + + + + + + + + + + + + Indicates that the test errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. Contains as a text node relevant data for the error, e.g., a stack trace + + + + + + + The error message. e.g., if a java exception is thrown, the return value of getMessage() + + + + + The type of error that occured. e.g., if a java execption is thrown the full class name of the exception. + + + + + + + + + Indicates that the test failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals. Contains as a text node relevant data for the failure, e.g., a stack trace + + + + + + + The message specified in the assert + + + + + The type of the assert. + + + + + + + + + + Name of the test method + + + + + Full class name for the class the test method is in. + + + + + Time taken (in seconds) to execute the test + + + + + + + Data that was written to standard out while the test was executed + + + + + + + + + + Data that was written to standard error while the test was executed + + + + + + + + + + + Full class name of the test for non-aggregated testsuite documents. Class name without the package for aggregated testsuites documents + + + + + + + + + + when the test was executed. Timezone may not be specified. + + + + + Host on which the tests were executed. 'localhost' should be used if the hostname cannot be determined. + + + + + + + + + + The total number of tests in the suite + + + + + The total number of tests in the suite that failed. A failure is a test which the code has explicitly failed by using the mechanisms for that purpose. e.g., via an assertEquals + + + + + The total number of tests in the suite that errored. An errored test is one that had an unanticipated problem. e.g., an unchecked throwable; or a problem with the implementation of the test. + + + + + The total number of ignored or skipped tests in the suite. + + + + + Time taken (in seconds) to execute the tests in the suite + + + + + + + + + diff --git a/unittests/test_policies.py b/unittests/test_policies.py index d03549a9f6..b553f29758 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -226,6 +226,9 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): with open(report_file, 'w') as fp: jsonext.dump(report, fp) + # Validate the junit report + runreport.load_xml_report(report_file) + # Read and validate the report using the runreport module runreport.load_report(report_file) @@ -466,7 +469,7 @@ def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats assert 10 == stats.num_cases(0) - assert 4 == len(stats.failed()) + assert 4 == len(stats.failed()) for tf in stats.failed(): check = tf.testcase.check _, exc_value, _ = tf.exc_info From 0cfdb0fe57172eb0016dc61bb5a4b2d8ba5c925b Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 22 Apr 2021 22:12:40 +0200 Subject: [PATCH 14/26] fix for review --- docs/manpage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index 6d4459bd1f..41eadd4208 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -276,7 +276,7 @@ Options controlling ReFrame output This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter. - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 ------------------------------------- From 54a971e6f096593b5b618fe1cb22ce58fdae6680 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 28 Apr 2021 00:10:50 +0200 Subject: [PATCH 15/26] Fine tune implementation --- reframe/frontend/cli.py | 21 +- reframe/frontend/runreport.py | 73 ++++--- reframe/frontend/statistics.py | 76 ------- reframe/schemas/config.json | 4 +- reframe/schemas/{JUnit.xsd => junit.xsd} | 0 unittests/test_junit.py | 239 ----------------------- unittests/test_policies.py | 14 +- 7 files changed, 66 insertions(+), 361 deletions(-) rename reframe/schemas/{JUnit.xsd => junit.xsd} (100%) delete mode 100644 unittests/test_junit.py diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index 4a32c47f2a..565fb6ef6d 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -34,7 +34,6 @@ from reframe.frontend.executors.policies import (SerialExecutionPolicy, AsynchronousExecutionPolicy) from reframe.frontend.executors import Runner, generate_testcases -from reframe.frontend.statistics import junit_lxml def format_check(check, check_deps, detailed=False): @@ -210,7 +209,7 @@ def main(): ) output_options.add_argument( '--report-junit', action='store', metavar='FILE', - help="Store XML junit run report in FILE", + help="Store a JUnit report in FILE", envvar='RFM_REPORT_JUNIT', configvar='general/report_junit' ) @@ -1029,19 +1028,17 @@ def module_unuse(*paths): ) # Generate the junit xml report for this session - report_xml = 'general/0/report_junit' - if site_config.get(report_xml) or os.getenv('RFM_REPORT_JUNIT'): - xml_report_file = os.path.normpath( - osext.expandvars(rt.get_option(report_xml)) - ) - xml_data = junit_lxml(json_report).decode() + junit_report_file = rt.get_option('general/0/report_junit') + if junit_report_file: + # Expand variables in filename + junit_report_file = osext.expandvars(junit_report_file) + junit_xml = runreport.junit_xml_report(json_report) try: - with open(xml_report_file, 'w') as fp: - fp.write(str(xml_data)) - fp.write('\n') + with open(junit_report_file, 'w') as fp: + runreport.junit_dump(junit_xml, fp) except OSError as e: printer.warning( - f'failed to generate report in {xml_report_file!r}:' + f'failed to generate report in {junit_report_file!r}: ' f'{e}' ) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index fe36ae48ac..4ff6bdd952 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -5,6 +5,7 @@ import json import jsonschema +import lxml.etree as etree import os import re @@ -13,13 +14,8 @@ import reframe.utility.jsonext as jsonext import reframe.utility.versioning as versioning -from io import BytesIO, StringIO -from lxml import etree -from reframe.frontend.statistics import junit_lxml - DATA_VERSION = '1.3.0' _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json') -_JUNIT_SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/JUnit.xsd') class _RunReport: @@ -158,31 +154,46 @@ def load_report(filename): return _RunReport(report) -def load_xml_report(filename): - try: - with open(filename, 'r') as fp: - json_report = json.load(fp) - except OSError as e: - raise errors.ReframeError( - f'failed to load report xml file {filename!r}') from e - except json.JSONDecodeError as e: - raise errors.ReframeError( - f'report file {filename!r} is not a valid JSON file') from e +def junit_xml_report(json_report): + '''Generate a JUnit report from a standard ReFrame JSON report.''' + + xml_testsuites = etree.Element('testsuites') + xml_testsuite = etree.SubElement( + xml_testsuites, 'testsuite', + attrib={ + 'name': 'rfm', + 'errors': '0', + 'failures': str(json_report['session_info']['num_failures']), + 'tests': str(json_report['session_info']['num_cases']), + 'time': str(json_report['session_info']['time_elapsed']), + 'hostname': json_report['session_info']['hostname'], + } + ) - # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd - f = open(_JUNIT_SCHEMA, 'rb') - try: - schema = etree.XMLSchema(file=f) - finally: - f.close() - - rfm_tree = etree.fromstring(junit_lxml(json_report)) - text = etree.tostring(rfm_tree) - f = BytesIO(text) if isinstance(text, bytes) else StringIO(text) - tree_valid = etree.parse(f) - try: - schema.validate(tree_valid) - except ValidationError as e: - raise errors.ReframeError(f'invalid junit report {filename!r}') from e + for testid in range(len(json_report['runs'][0]['testcases'])): + tid = json_report['runs'][0]['testcases'][testid] + casename = ( + f"{tid['name']}[{tid['system']}, {tid['environment']}]" + ) + testcase = etree.SubElement( + xml_testsuite, 'testcase', + attrib={ + 'classname': tid['filename'], + 'name': casename, + 'time': str(tid['time_total']), + } + ) + if tid['result'] == 'failure': + testcase_msg = etree.SubElement( + testcase, 'failure', attrib={'type': tid['fail_phase']} + ) + testcase_msg.text = tid['fail_reason'] + + return xml_testsuites - return _RunReport(json_report) + +def junit_dump(xml, fp): + fp.write( + etree.tostring(xml, encoding='utf8', pretty_print=True, + method='xml', xml_declaration=True).decode() + ) diff --git a/reframe/frontend/statistics.py b/reframe/frontend/statistics.py index 0151ae70d3..5ee6a757d5 100644 --- a/reframe/frontend/statistics.py +++ b/reframe/frontend/statistics.py @@ -5,8 +5,6 @@ import inspect import traceback -import xml.etree.ElementTree as ET -import lxml.etree as LX import reframe.core.runtime as rt import reframe.core.exceptions as errors @@ -327,77 +325,3 @@ def performance_report(self): report_end]) return '' - - -def junit_xml(json_report): - # https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd - xml_testsuites = ET.Element('testsuites') - xml_testsuite = ET.SubElement( - xml_testsuites, 'testsuite', - attrib={ - 'name': 'rfm', - 'errors': '0', - 'failures': str(json_report['session_info']['num_failures']), - 'tests': str(json_report['session_info']['num_cases']), - 'time': str(json_report['session_info']['time_elapsed']), - 'hostname': json_report['session_info']['hostname'], - } - ) - - for testid in range(len(json_report['runs'][0]['testcases'])): - tid = json_report['runs'][0]['testcases'][testid] - casename = ( - f"{tid['name']}[{tid['system']}, {tid['environment']}]" - ) - testcase = ET.SubElement( - xml_testsuite, 'testcase', - attrib={ - 'classname': tid['filename'], - 'name': casename, - 'time': str(tid['time_total']), - } - ) - if tid['result'] == 'failure': - testcase_msg = ET.SubElement( - testcase, 'failure', attrib={'type': tid['fail_phase']} - ) - testcase_msg.text = tid['fail_reason'] - - return ET.tostring(xml_testsuites, encoding='utf8', method='xml') - -def junit_lxml(json_report): - # https://lxml.de/tutorial.html - xml_testsuites = LX.Element('testsuites') - xml_testsuite = LX.SubElement( - xml_testsuites, 'testsuite', - attrib={ - 'name': 'rfm', - 'errors': '0', - 'failures': str(json_report['session_info']['num_failures']), - 'tests': str(json_report['session_info']['num_cases']), - 'time': str(json_report['session_info']['time_elapsed']), - 'hostname': json_report['session_info']['hostname'], - } - ) - - for testid in range(len(json_report['runs'][0]['testcases'])): - tid = json_report['runs'][0]['testcases'][testid] - casename = ( - f"{tid['name']}[{tid['system']}, {tid['environment']}]" - ) - testcase = LX.SubElement( - xml_testsuite, 'testcase', - attrib={ - 'classname': tid['filename'], - 'name': casename, - 'time': str(tid['time_total']), - } - ) - if tid['result'] == 'failure': - testcase_msg = LX.SubElement( - testcase, 'failure', attrib={'type': tid['fail_phase']} - ) - testcase_msg.text = tid['fail_reason'] - - return LX.tostring(xml_testsuites, encoding='utf8', pretty_print=True, - method='xml', xml_declaration=True) diff --git a/reframe/schemas/config.json b/reframe/schemas/config.json index 1e35804581..e63c0203fb 100644 --- a/reframe/schemas/config.json +++ b/reframe/schemas/config.json @@ -448,7 +448,7 @@ "non_default_craype": {"type": "boolean"}, "purge_environment": {"type": "boolean"}, "report_file": {"type": "string"}, - "report_junit": {"type": "string"}, + "report_junit": {"type": ["string", "null"]}, "save_log_files": {"type": "boolean"}, "target_systems": {"$ref": "#/defs/system_ref"}, "timestamp_dirs": {"type": "string"}, @@ -487,7 +487,7 @@ "general/non_default_craype": false, "general/purge_environment": false, "general/report_file": "${HOME}/.reframe/reports/run-report.json", - "general/report_junit": "", + "general/report_junit": null, "general/save_log_files": false, "general/target_systems": ["*"], "general/timestamp_dirs": "", diff --git a/reframe/schemas/JUnit.xsd b/reframe/schemas/junit.xsd similarity index 100% rename from reframe/schemas/JUnit.xsd rename to reframe/schemas/junit.xsd diff --git a/unittests/test_junit.py b/unittests/test_junit.py deleted file mode 100644 index b970be75f2..0000000000 --- a/unittests/test_junit.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import os -import pytest -import sys - -from lxml import etree -from lxml.doctestcompare import norm_whitespace -from reframe.frontend.statistics import junit_lxml - -# https://github.com/SuminAndrew/lxml-asserts/blob/master/LICENSE - - -def _describe_element(elem): - return elem.getroottree().getpath(elem) - - -def _xml_compare_text(t1, t2, strip): - t1 = t1 or '' - t2 = t2 or '' - - if strip: - t1 = norm_whitespace(t1).strip() - t2 = norm_whitespace(t2).strip() - - return t1 == t2 - - -def _assert_tag_and_attributes_are_equal(xml1, xml2, can_extend=False): - if xml1.tag != xml2.tag: - raise AssertionError( - u"Tags do not match: {tag1} != {tag2}".format( - tag1=_describe_element(xml1), tag2=_describe_element(xml2) - ) - ) - - added_attributes = set(xml2.attrib).difference(xml1.attrib) - missing_attributes = set(xml1.attrib).difference(xml2.attrib) - - if missing_attributes: - raise AssertionError( - u"Second xml misses attributes: {path}/({attributes})".format( - path=_describe_element(xml2), attributes=','.join( - missing_attributes) - ) - ) - - if not can_extend and added_attributes: - raise AssertionError( - (u"Second xml has additional attributes: " - u"{path}/({attributes})".format( - path=_describe_element(xml2), - attributes=','.join(added_attributes))) - ) - - for attrib in xml1.attrib: - if not _xml_compare_text(xml1.attrib[attrib], xml2.attrib[attrib], - False): - raise AssertionError( - (u"Attribute values are not equal: {path}/{attribute}" - u"['{v1}' != '{v2}']".format( - path=_describe_element(xml1), - attribute=attrib, - v1=xml1.attrib[attrib], - v2=xml2.attrib[attrib]))) - - if not _xml_compare_text(xml1.text, xml2.text, True): - raise AssertionError( - u"Tags text differs: {path}['{t1}' != '{t2}']".format( - path=_describe_element(xml1), t1=xml1.text, t2=xml2.text - ) - ) - - if not _xml_compare_text(xml1.tail, xml2.tail, True): - raise AssertionError( - u"Tags tail differs: {path}['{t1}' != '{t2}']".format( - path=_describe_element(xml1), t1=xml1.tail, t2=xml2.tail - ) - ) - - -def _assert_xml_docs_are_equal(xml1, xml2, check_tags_order=False): - _assert_tag_and_attributes_are_equal(xml1, xml2) - - children1 = list(xml1) - children2 = list(xml2) - - if len(children1) != len(children2): - raise AssertionError( - (u"Children are not equal: " - u"{len1} children != {len2} children]".format( - path=_describe_element(xml1), len1=len(children1), - len2=len(children2)))) - - raise AssertionError( - (u"Children are not equal: " - u"{path}[{len1} children != {len2} children]".format( - path=_describe_element(xml1), len1=len(children1), - len2=len(children2)))) - - if check_tags_order: - for c1, c2 in zip(children1, children2): - _assert_xml_docs_are_equal(c1, c2, True) - - else: - children1 = set(children1) - children2 = set(children2) - - for c1 in children1: - c1_match = None - - for c2 in children2: - try: - _assert_xml_docs_are_equal(c1, c2, False) - except AssertionError: - pass - else: - c1_match = c2 - break - - if c1_match is None: - raise AssertionError( - u"No equal child found in second xml: {path}".format( - path=_describe_element(c1) - ) - ) - - children2.remove(c1_match) - - -def _assert_xml_compare(cmp_func, xml1, xml2, **kwargs): - if not isinstance(xml1, etree._Element): - xml1 = etree.fromstring(xml1) - - if not isinstance(xml2, etree._Element): - xml2 = etree.fromstring(xml2) - - cmp_func(xml1, xml2, **kwargs) - - -def assert_xml_equal(first, second, check_tags_order=False): - _assert_xml_compare( - _assert_xml_docs_are_equal, first, second, - check_tags_order=check_tags_order - ) - - -def _generate_json(): - json_rpt = { - 'session_info': { - 'num_failures': '1', - 'num_cases': '2', - 'time_elapsed': '0.445', - 'hostname': 'dom101', - }, - 'runs': [ - { - 'testcases': [ - { - 'name': 'P100_Test', - 'system': 'dom:mc', - 'time_total': '0.179', - 'filename': 'test.py', - 'environment': 'PrgEnv-cray', - 'result': 'success', - }, - { - 'name': 'V100_Test', - 'system': 'dom:mc', - 'time_total': '0.266', - 'filename': 'test.py', - 'environment': 'PrgEnv-cray', - 'result': 'failure', - 'fail_phase': 'sanity', - 'fail_reason': ( - "sanity error: pattern 'x' not found in " - "'rfm_V100_Test_1_job.out'" - ), - }, - ], - } - ], - } - return json_rpt - - -def assertXmlEqual(self, first, second, check_tags_order=False, msg=None): - ''' - Assert that two xml documents are equal. - :param first: first etree object or xml string - :param second: second etree object or xml string - :param check_tags_order: if False, the order of children is ignored - :param msg: custom error message - :return: raises failureException if xml documents are not equal - ''' - if msg is None: - msg = u'XML documents are not equal' - - try: - assert_xml_equal(first, second, check_tags_order) - except AssertionError as e: - raise_exc_info( - ( - self.failureException, - self.failureException(u"{} — {}".format(msg, unicode_type(e))), - sys.exc_info()[2], - ) - ) - - -def test_xmlreport(): - # - reference_tree_str = """ - - - - - sanity error: pattern 'x' not found in - 'rfm_V100_Test_1_job.out' - - - - - """.strip() - reference_tree = etree.fromstring(reference_tree_str) - json_report = _generate_json() - rfm_tree = etree.fromstring(junit_lxml(json_report)) - # debug with: print(etree.tostring(rfm_tree).decode('utf-8')) - msg = u'XML documents are not equal' - try: - assert_xml_equal(rfm_tree, reference_tree, check_tags_order=False) - except AssertionError as e: - print(f'___{e}___') diff --git a/unittests/test_policies.py b/unittests/test_policies.py index b553f29758..8efca5394b 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -21,6 +21,7 @@ import reframe.utility.osext as osext import unittests.utility as test_util +from lxml import etree from reframe.core.exceptions import (AbortTaskError, FailureLimitError, ReframeError, @@ -178,6 +179,16 @@ def _validate_runreport(report): jsonschema.validate(json.loads(report), schema) +def _validate_junit_report(report): + # Cloned from + # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd + schema_file = 'reframe/schemas/junit.xsd' + with open(schema_file) as fp: + schema = etree.XMLSchema(etree.parse(fp)) + + schema.assert_(report) + + def _generate_runreport(run_stats, time_start, time_end): return { 'session_info': { @@ -227,7 +238,8 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): jsonext.dump(report, fp) # Validate the junit report - runreport.load_xml_report(report_file) + xml_report = runreport.junit_xml_report(report) + _validate_junit_report(xml_report) # Read and validate the report using the runreport module runreport.load_report(report_file) From c504ded7b873ea500717d33eb09c024204d0fcc4 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 28 Apr 2021 00:12:25 +0200 Subject: [PATCH 16/26] Style change --- unittests/test_policies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 8efca5394b..00022cadc9 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -481,7 +481,7 @@ def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats assert 10 == stats.num_cases(0) - assert 4 == len(stats.failed()) + assert 4 == len(stats.failed()) for tf in stats.failed(): check = tf.testcase.check _, exc_value, _ = tf.exc_info From d22ea80384bcdb08b61c85197014debd421d38a2 Mon Sep 17 00:00:00 2001 From: jgp Date: Wed, 28 Apr 2021 20:19:33 +0200 Subject: [PATCH 17/26] fix for review --- reframe/frontend/runreport.py | 50 ++++++++++++++++++++++++++++++----- unittests/test_policies.py | 3 ++- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 4ff6bdd952..db45a5cb53 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +import dateutil.parser as parser import json import jsonschema import lxml.etree as etree @@ -158,18 +159,24 @@ def junit_xml_report(json_report): '''Generate a JUnit report from a standard ReFrame JSON report.''' xml_testsuites = etree.Element('testsuites') + isotime = parser.parse(json_report['session_info']['time_start'][:-6]) xml_testsuite = etree.SubElement( xml_testsuites, 'testsuite', attrib={ - 'name': 'rfm', 'errors': '0', 'failures': str(json_report['session_info']['num_failures']), + 'hostname': json_report['session_info']['hostname'], + 'id': '0', + 'name': 'rfm', + 'package': 'rfm', 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), - 'hostname': json_report['session_info']['hostname'], + 'timestamp': isotime.isoformat(), } ) - + testsuite_properties = etree.SubElement(xml_testsuite, 'properties') + # etree.SubElement(testsuite_properties, "property", + # {'name': 'x', 'value': '0'}) for testid in range(len(json_report['runs'][0]['testcases'])): tid = json_report['runs'][0]['testcases'][testid] casename = ( @@ -185,10 +192,41 @@ def junit_xml_report(json_report): ) if tid['result'] == 'failure': testcase_msg = etree.SubElement( - testcase, 'failure', attrib={'type': tid['fail_phase']} + testcase, 'failure', attrib={'type': 'failure', + 'message': tid['fail_phase']} ) - testcase_msg.text = tid['fail_reason'] - + testcase_msg.text = f"{tid['fail_phase']}: {tid['fail_reason']}" + + testsuite_stdo = etree.SubElement(xml_testsuite, 'system-out') + testsuite_stdo.text = '' + testsuite_stde = etree.SubElement(xml_testsuite, 'system-err') + testsuite_stde.text = '' + + # --- + # testcase_error = etree.SubElement( + # xml_testsuite, 'testcase', + # attrib={'classname': 'rfmE', 'name': 'test error', 'time': '0'} + # ) + # testcase_error_msg = etree.SubElement( + # testcase_error, 'error', + # attrib={'message': 'no test error', 'type': 'error'} + # ) + # testcase_error_msg.text = 'E' + # --- + # testcase_skip = etree.SubElement( + # xml_testsuite, 'testcase', + # attrib={'classname': 'rfmS', 'name': 'test skip', 'time': '0'} + # ) + # testcase_skip_msg = etree.SubElement( + # testcase_skip, 'skipped', + # attrib={'message': 'no test skipped', 'type': 'skipped'} + # ) + # testcase_skip_msg.text = 'S' + + # debug_str = etree.tostring(xml_testsuites, encoding='utf8', + # pretty_print=True, method='xml', + # xml_declaration=True).decode() + # print(f'xml_testsuites={debug_str}') return xml_testsuites diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 00022cadc9..ec7ae86ca4 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -239,6 +239,7 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): # Validate the junit report xml_report = runreport.junit_xml_report(report) + # print(f'xml_report={xml_report}') _validate_junit_report(xml_report) # Read and validate the report using the runreport module @@ -481,7 +482,7 @@ def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats assert 10 == stats.num_cases(0) - assert 4 == len(stats.failed()) + assert 4 == len(stats.failed()) for tf in stats.failed(): check = tf.testcase.check _, exc_value, _ = tf.exc_info From 1ce77054856b4a880ea5b3f7f6b6a6f09ab4cdad Mon Sep 17 00:00:00 2001 From: jgp Date: Wed, 28 Apr 2021 20:22:39 +0200 Subject: [PATCH 18/26] import --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index b5040974ec..e5da2c027d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ argcomplete==1.12.2 coverage==5.5 +dateutil importlib_metadata==3.10.0; python_version < '3.8' jsonschema==3.2.0 lxml==4.6.3 From 024281130c860b130b5cda041ed6869691a6379d Mon Sep 17 00:00:00 2001 From: jgp Date: Wed, 28 Apr 2021 20:26:44 +0200 Subject: [PATCH 19/26] fix for review --- reframe/frontend/runreport.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index db45a5cb53..3f1032350b 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -167,8 +167,8 @@ def junit_xml_report(json_report): 'failures': str(json_report['session_info']['num_failures']), 'hostname': json_report['session_info']['hostname'], 'id': '0', - 'name': 'rfm', - 'package': 'rfm', + 'name': 'reframe', + 'package': 'reframe', 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), 'timestamp': isotime.isoformat(), From be61e07b111671bda0e91339a59bd256c8626e6c Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 29 Apr 2021 15:11:01 +0200 Subject: [PATCH 20/26] fix for review --- reframe/frontend/runreport.py | 16 +++++----------- requirements.txt | 1 - unittests/test_policies.py | 3 +-- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 3f1032350b..ad77021c8d 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -3,7 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -import dateutil.parser as parser import json import jsonschema import lxml.etree as etree @@ -159,7 +158,6 @@ def junit_xml_report(json_report): '''Generate a JUnit report from a standard ReFrame JSON report.''' xml_testsuites = etree.Element('testsuites') - isotime = parser.parse(json_report['session_info']['time_start'][:-6]) xml_testsuite = etree.SubElement( xml_testsuites, 'testsuite', attrib={ @@ -171,7 +169,7 @@ def junit_xml_report(json_report): 'package': 'reframe', 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), - 'timestamp': isotime.isoformat(), + 'timestamp': json_report['session_info']['time_start'][:-5], } ) testsuite_properties = etree.SubElement(xml_testsuite, 'properties') @@ -197,10 +195,10 @@ def junit_xml_report(json_report): ) testcase_msg.text = f"{tid['fail_phase']}: {tid['fail_reason']}" - testsuite_stdo = etree.SubElement(xml_testsuite, 'system-out') - testsuite_stdo.text = '' - testsuite_stde = etree.SubElement(xml_testsuite, 'system-err') - testsuite_stde.text = '' + testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out') + testsuite_stdout.text = '' + testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err') + testsuite_stderr.text = '' # --- # testcase_error = etree.SubElement( @@ -223,10 +221,6 @@ def junit_xml_report(json_report): # ) # testcase_skip_msg.text = 'S' - # debug_str = etree.tostring(xml_testsuites, encoding='utf8', - # pretty_print=True, method='xml', - # xml_declaration=True).decode() - # print(f'xml_testsuites={debug_str}') return xml_testsuites diff --git a/requirements.txt b/requirements.txt index e5da2c027d..b5040974ec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ argcomplete==1.12.2 coverage==5.5 -dateutil importlib_metadata==3.10.0; python_version < '3.8' jsonschema==3.2.0 lxml==4.6.3 diff --git a/unittests/test_policies.py b/unittests/test_policies.py index ec7ae86ca4..66fc2c1b30 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -239,7 +239,6 @@ def test_runall(make_runner, make_cases, common_exec_ctx, tmp_path): # Validate the junit report xml_report = runreport.junit_xml_report(report) - # print(f'xml_report={xml_report}') _validate_junit_report(xml_report) # Read and validate the report using the runreport module @@ -482,7 +481,7 @@ def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats assert 10 == stats.num_cases(0) - assert 4 == len(stats.failed()) + assert 4 == len(stats.failed()) for tf in stats.failed(): check = tf.testcase.check _, exc_value, _ = tf.exc_info From d8cfd713e1d8b9a72161f2c92228722dd18e9ec2 Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 29 Apr 2021 15:35:36 +0200 Subject: [PATCH 21/26] fix for UnicodeDecodeError --- reframe/schemas/junit.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/schemas/junit.xsd b/reframe/schemas/junit.xsd index 84b0f157b1..44637c4855 100644 --- a/reframe/schemas/junit.xsd +++ b/reframe/schemas/junit.xsd @@ -5,7 +5,7 @@ attributeFormDefault="unqualified"> JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks -Copyright © 2011, Windy Road Technology Pty. Limited +Copyright 2011, Windy Road Technology Pty. Limited The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). From 631d08b0e6ff77827729af52f06a7ed249df87de Mon Sep 17 00:00:00 2001 From: jgp Date: Thu, 29 Apr 2021 15:44:07 +0200 Subject: [PATCH 22/26] fix for error type 'xs:decimal' --- reframe/frontend/runreport.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index ad77021c8d..8940ac9311 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: BSD-3-Clause +import decimal import json import jsonschema import lxml.etree as etree @@ -185,7 +186,7 @@ def junit_xml_report(json_report): attrib={ 'classname': tid['filename'], 'name': casename, - 'time': str(tid['time_total']), + 'time': str(decimal.Decimal(tid['time_total'])), } ) if tid['result'] == 'failure': From 42285eb76b53a170aad931836f74a53885414043 Mon Sep 17 00:00:00 2001 From: jgp Date: Sat, 1 May 2021 14:02:48 +0200 Subject: [PATCH 23/26] fix for review --- reframe/schemas/junit.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/schemas/junit.xsd b/reframe/schemas/junit.xsd index 44637c4855..84b0f157b1 100644 --- a/reframe/schemas/junit.xsd +++ b/reframe/schemas/junit.xsd @@ -5,7 +5,7 @@ attributeFormDefault="unqualified"> JUnit test result schema for the Apache Ant JUnit and JUnitReport tasks -Copyright 2011, Windy Road Technology Pty. Limited +Copyright © 2011, Windy Road Technology Pty. Limited The Apache Ant JUnit XML Schema is distributed under the terms of the Apache License Version 2.0 http://www.apache.org/licenses/ Permission to waive conditions of this license may be requested from Windy Road Support (http://windyroad.org/support). From 6d31bb74720d20444bbbaa403ce5d2ac0a4cfff8 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 2 May 2021 21:10:51 +0200 Subject: [PATCH 24/26] Final fine tuning --- docs/config_reference.rst | 7 ++++--- docs/manpage.rst | 25 +++++++++++++++++++------ reframe/frontend/runreport.py | 30 ++++++------------------------ unittests/test_policies.py | 4 ++-- 4 files changed, 31 insertions(+), 35 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 49d8d0b0eb..31fbf54a0e 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -1227,11 +1227,12 @@ General Configuration .. js:attribute:: .general[].report_junit :required: No - :default: ``"${HOME}/.reframe/reports/run-report.xml"`` + :default: ``null`` - The file where ReFrame will store its report in junit xml format. + The file where ReFrame will store its report in JUnit format. + The report adheres to the XSD schema `here `__. - .. versionadded:: 3.6 + .. versionadded:: 3.6.0 .. js:attribute:: .general[].resolve_module_conflicts diff --git a/docs/manpage.rst b/docs/manpage.rst index 23bd45d02f..cf8b0de8e9 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -276,13 +276,11 @@ Options controlling ReFrame output .. option:: --report-junit=FILE - The file where ReFrame will store its report in junit xml format. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, - in which case ReFrame will generate a new report each time it is run by - appending a counter to the report file. + Instruct ReFrame to generate a JUnit XML report in ``FILE``. + The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. + The generated report adheres to the XSD schema `here `__. - This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment - variable or the :js:attr:`report_junit` general configuration parameter. + This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter. .. versionadded:: 3.6.0 @@ -872,6 +870,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame: ================================== ================== +.. envvar:: RFM_REPORT_JUNIT + + The file where ReFrame will generate a JUnit XML report. + + .. versionadded:: 3.6.0 + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--report-junit` + Associated configuration parameter :js:attr:`report_junit` general configuration parameter + ================================== ================== + + .. envvar:: RFM_RESOLVE_MODULE_CONFLICTS Resolve module conflicts automatically. diff --git a/reframe/frontend/runreport.py b/reframe/frontend/runreport.py index 8940ac9311..3df8eaf675 100644 --- a/reframe/frontend/runreport.py +++ b/reframe/frontend/runreport.py @@ -170,12 +170,12 @@ def junit_xml_report(json_report): 'package': 'reframe', 'tests': str(json_report['session_info']['num_cases']), 'time': str(json_report['session_info']['time_elapsed']), + + # XSD schema does not like the timezone format, so we remove it 'timestamp': json_report['session_info']['time_start'][:-5], } ) testsuite_properties = etree.SubElement(xml_testsuite, 'properties') - # etree.SubElement(testsuite_properties, "property", - # {'name': 'x', 'value': '0'}) for testid in range(len(json_report['runs'][0]['testcases'])): tid = json_report['runs'][0]['testcases'][testid] casename = ( @@ -186,6 +186,10 @@ def junit_xml_report(json_report): attrib={ 'classname': tid['filename'], 'name': casename, + + # XSD schema does not like the exponential format and since we + # do not want to impose a fixed width, we pass it to `Decimal` + # to format it automatically. 'time': str(decimal.Decimal(tid['time_total'])), } ) @@ -200,28 +204,6 @@ def junit_xml_report(json_report): testsuite_stdout.text = '' testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err') testsuite_stderr.text = '' - - # --- - # testcase_error = etree.SubElement( - # xml_testsuite, 'testcase', - # attrib={'classname': 'rfmE', 'name': 'test error', 'time': '0'} - # ) - # testcase_error_msg = etree.SubElement( - # testcase_error, 'error', - # attrib={'message': 'no test error', 'type': 'error'} - # ) - # testcase_error_msg.text = 'E' - # --- - # testcase_skip = etree.SubElement( - # xml_testsuite, 'testcase', - # attrib={'classname': 'rfmS', 'name': 'test skip', 'time': '0'} - # ) - # testcase_skip_msg = etree.SubElement( - # testcase_skip, 'skipped', - # attrib={'message': 'no test skipped', 'type': 'skipped'} - # ) - # testcase_skip_msg.text = 'S' - return xml_testsuites diff --git a/unittests/test_policies.py b/unittests/test_policies.py index 66fc2c1b30..52f462e85e 100644 --- a/unittests/test_policies.py +++ b/unittests/test_policies.py @@ -183,7 +183,7 @@ def _validate_junit_report(report): # Cloned from # https://raw.githubusercontent.com/windyroad/JUnit-Schema/master/JUnit.xsd schema_file = 'reframe/schemas/junit.xsd' - with open(schema_file) as fp: + with open(schema_file, encoding='utf-8') as fp: schema = etree.XMLSchema(etree.parse(fp)) schema.assert_(report) @@ -481,7 +481,7 @@ def assert_dependency_run(runner): assert_runall(runner) stats = runner.stats assert 10 == stats.num_cases(0) - assert 4 == len(stats.failed()) + assert 4 == len(stats.failed()) for tf in stats.failed(): check = tf.testcase.check _, exc_value, _ = tf.exc_info From a30f9c2357e6d4844f7997f03b22ea6e92d96dec Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 2 May 2021 21:20:20 +0200 Subject: [PATCH 25/26] Fix wheel package creation --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2013fc593a..b698fc23a3 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ ), package_data={'reframe': ['schemas/*']}, include_package_data=True, - install_requires=['argcomplete', 'jsonschema', 'PyYAML', 'semver'], + install_requires=['argcomplete', 'jsonschema', 'lxml', 'PyYAML', 'semver'], python_requires='>=3.6', scripts=['bin/reframe'], classifiers=( From da5a06727a5b9e17fb31eda8091935e08a2e1ae7 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 4 May 2021 22:47:14 +0200 Subject: [PATCH 26/26] Final corrections --- docs/manpage.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/manpage.rst b/docs/manpage.rst index cf8b0de8e9..bc95f0307d 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -277,8 +277,7 @@ Options controlling ReFrame output .. option:: --report-junit=FILE Instruct ReFrame to generate a JUnit XML report in ``FILE``. - The ``FILE`` argument may contain the special placeholder ``{sessionid}``, in which case ReFrame will generate a new report each time it is run by appending a counter to the report file. - The generated report adheres to the XSD schema `here `__. + The generated report adheres to the XSD schema `here `__ and it takes into account only the first run, ignoring retries of failed tests. This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter.