diff --git a/docs/commands/test.rst b/docs/commands/test.rst index 21a3fe0a1..5177d9f5b 100644 --- a/docs/commands/test.rst +++ b/docs/commands/test.rst @@ -35,6 +35,7 @@ please careful and do not try this against production Galaxy instances. --test_output PATH Output test report (HTML - for humans). --test_output_xunit PATH Output test report (xUnit style - for computers). + --test_output_json PATH Output test report (planemo json). --job_output_files DIRECTORY Write job outputs to specified directory. --update_test_data Update test-data directory with job outputs (normally written to directory diff --git a/docs/planemo.rst b/docs/planemo.rst index 464d9534e..bb114c6a8 100644 --- a/docs/planemo.rst +++ b/docs/planemo.rst @@ -43,6 +43,14 @@ planemo.galaxy_run module :undoc-members: :show-inheritance: +planemo.galaxy_test module +-------------------------- + +.. automodule:: planemo.galaxy_test + :members: + :undoc-members: + :show-inheritance: + planemo.io module ----------------- diff --git a/planemo/commands/cmd_test.py b/planemo/commands/cmd_test.py index e615a5275..841988abe 100644 --- a/planemo/commands/cmd_test.py +++ b/planemo/commands/cmd_test.py @@ -1,7 +1,5 @@ -import json import os import sys -import xml.etree.ElementTree as ET import click @@ -10,13 +8,16 @@ from planemo import options from planemo import galaxy_config from planemo import galaxy_run +from planemo import galaxy_test from galaxy.tools.deps.commands import shell XUNIT_UPGRADE_MESSAGE = ("This version of Galaxy does not support xUnit - " "please update to newest development brach.") -NO_XUNIT_MESSAGE = ("Cannot locate xUnit report for tests - update to a new " +NO_XUNIT_MESSAGE = ("Cannot locate xUnit report option for tests - update " "Galaxy for more detailed breakdown.") +NO_JSON_MESSAGE = ("Cannot locate json report option for tests - update " + "Galaxy for more detailed breakdown.") NO_TESTS_MESSAGE = "No tests were executed - see Galaxy output for details." ALL_TESTS_PASSED_MESSAGE = "All %d test(s) executed passed." PROBLEM_COUNT_MESSAGE = ("There were problems with %d test(s) - out of %d " @@ -45,6 +46,12 @@ help="Output test report (xUnit style - for computers).", default=None, ) +@click.option( + "--test_output_json", + type=click.Path(file_okay=True, resolve_path=True), + help="Output test report (planemo json).", + default=None, +) @click.option( "--job_output_files", type=click.Path(file_okay=False, resolve_path=True), @@ -169,24 +176,17 @@ def __summarize_tests_full( structured_report_file, **kwds ): - try: - structured_data = json.load(open(structured_report_file, "r"))["tests"] - except Exception: - # Older Galaxy's will not support this option. - structured_data = {} - - xunit_tree = ET.parse(xunit_report_file) - xunit_root = xunit_tree.getroot() - xunit_attrib = xunit_root.attrib - num_tests = int(xunit_attrib.get("tests", 0)) - num_failures = int(xunit_attrib.get("failures", 0)) - num_errors = int(xunit_attrib.get("errors", 0)) - num_skips = int(xunit_attrib.get("skips", 0)) + test_results = galaxy_test.GalaxyTestResults( + structured_report_file, + xunit_report_file + ) + num_tests = test_results.num_tests + num_problems = test_results.num_problems + if num_tests == 0: warn(NO_TESTS_MESSAGE) return - num_problems = num_skips + num_errors + num_failures if num_problems == 0: info(ALL_TESTS_PASSED_MESSAGE % num_tests) @@ -195,8 +195,9 @@ def __summarize_tests_full( message = PROBLEM_COUNT_MESSAGE % message_args warn(message) - for testcase_el in xunit_root.findall("testcase"): - __summarize_test_case(structured_data, testcase_el) + for testcase_el in test_results.xunit_testcase_elements: + structured_data_tests = test_results.structured_data_tests + __summarize_test_case(structured_data_tests, testcase_el) def __summarize_test_case(structured_data, testcase_el, **kwds): @@ -279,8 +280,12 @@ def __structured_report_file(kwds, config): structured_data_supported = False structured_report_file = None - if structured_data_supported: + structured_report_file = kwds.get("test_output_json", None) + if structured_report_file is None and structured_data_supported: conf_dir = config.config_directory - structured_report_file = os.path.join(conf_dir, "structured_data.xml") + structured_report_file = os.path.join(conf_dir, "structured_data.json") + elif structured_report_file is not None and not structured_data_supported: + warn(NO_JSON_MESSAGE) + structured_report_file = None return structured_report_file diff --git a/planemo/galaxy_test.py b/planemo/galaxy_test.py new file mode 100644 index 000000000..c2545cd0c --- /dev/null +++ b/planemo/galaxy_test.py @@ -0,0 +1,59 @@ +""" Utilities for reasoning about Galaxy test results. +""" +import json +import xml.etree.ElementTree as ET + + +class GalaxyTestResults(object): + """ Class that combine the test-centric xunit output + with the Galaxy centric structured data output - and + abstracts away the difference (someday). + """ + + def __init__(self, output_json_path, output_xml_path): + try: + output_json_f = open(output_json_path, "r") + structured_data = json.load(output_json_f) + structured_data_tests = structured_data["tests"] + except Exception: + # Older Galaxy's will not support this option. + structured_data = {} + structured_data_tests = {} + self.structured_data = structured_data + self.structured_data_tests = structured_data_tests + self.xunit_tree = ET.parse(output_xml_path) + self.__merge_xunit() + try: + json.dump(self.structured_data, open(output_json_path, "w")) + except Exception: + pass + + @property + def _xunit_root(self): + return self.xunit_tree.getroot() + + def __merge_xunit(self): + xunit_attrib = self._xunit_root.attrib + num_tests = int(xunit_attrib.get("tests", 0)) + num_failures = int(xunit_attrib.get("failures", 0)) + num_errors = int(xunit_attrib.get("errors", 0)) + num_skips = int(xunit_attrib.get("skips", 0)) + summary = dict( + num_tests=num_tests, + num_failures=num_failures, + num_errors=num_errors, + num_skips=num_skips, + ) + + self.structured_data["summary"] = summary + self.num_tests = num_tests + self.num_problems = num_skips + num_errors + num_failures + + @property + def all_tests_passed(self): + return self.num_problems == 0 + + @property + def xunit_testcase_elements(self): + for testcase_el in self._xunit_root.findall("testcase"): + yield testcase_el