Expose output json option via CLI - refactoring.

This output file contains a fairly detailed breakdown of test results including stuff like command line, standard outputs, standard error, exit code, job metrics, etc....

docs/commands/test.rst

@@ -35,6 +35,7 @@ please careful and do not try this against production Galaxy instances.
       --test_output PATH              Output test report (HTML - for humans).
       --test_output_xunit PATH        Output test report (xUnit style - for
                                       computers).
+      --test_output_json PATH         Output test report (planemo json).
       --job_output_files DIRECTORY    Write job outputs to specified directory.
       --update_test_data              Update test-data directory with job outputs
                                       (normally written to directory

docs/planemo.rst

@@ -43,6 +43,14 @@ planemo.galaxy_run module
     :undoc-members:
     :show-inheritance:
 
+planemo.galaxy_test module
+--------------------------
+
+.. automodule:: planemo.galaxy_test
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 planemo.io module
 -----------------
 

planemo/commands/cmd_test.py

@@ -1,7 +1,5 @@
-import json
 import os
 import sys
-import xml.etree.ElementTree as ET
 
 import click
 
@@ -10,13 +8,16 @@
 from planemo import options
 from planemo import galaxy_config
 from planemo import galaxy_run
+from planemo import galaxy_test
 
 from galaxy.tools.deps.commands import shell
 
 XUNIT_UPGRADE_MESSAGE = ("This version of Galaxy does not support xUnit - "
                          "please update to newest development brach.")
-NO_XUNIT_MESSAGE = ("Cannot locate xUnit report for tests - update to a new "
+NO_XUNIT_MESSAGE = ("Cannot locate xUnit report option for tests - update "
                     "Galaxy for more detailed breakdown.")
+NO_JSON_MESSAGE = ("Cannot locate json report option for tests - update "
+                   "Galaxy for more detailed breakdown.")
 NO_TESTS_MESSAGE = "No tests were executed - see Galaxy output for details."
 ALL_TESTS_PASSED_MESSAGE = "All %d test(s) executed passed."
 PROBLEM_COUNT_MESSAGE = ("There were problems with %d test(s) - out of %d "
@@ -45,6 +46,12 @@
     help="Output test report (xUnit style - for computers).",
     default=None,
 )
+@click.option(
+    "--test_output_json",
+    type=click.Path(file_okay=True, resolve_path=True),
+    help="Output test report (planemo json).",
+    default=None,
+)
 @click.option(
     "--job_output_files",
     type=click.Path(file_okay=False, resolve_path=True),
@@ -169,24 +176,17 @@ def __summarize_tests_full(
     structured_report_file,
     **kwds
 ):
-    try:
-        structured_data = json.load(open(structured_report_file, "r"))["tests"]
-    except Exception:
-        # Older Galaxy's will not support this option.
-        structured_data = {}
-
-    xunit_tree = ET.parse(xunit_report_file)
-    xunit_root = xunit_tree.getroot()
-    xunit_attrib = xunit_root.attrib
-    num_tests = int(xunit_attrib.get("tests", 0))
-    num_failures = int(xunit_attrib.get("failures", 0))
-    num_errors = int(xunit_attrib.get("errors", 0))
-    num_skips = int(xunit_attrib.get("skips", 0))
+    test_results = galaxy_test.GalaxyTestResults(
+        structured_report_file,
+        xunit_report_file
+    )
+    num_tests = test_results.num_tests
+    num_problems = test_results.num_problems
+
     if num_tests == 0:
         warn(NO_TESTS_MESSAGE)
         return
 
-    num_problems = num_skips + num_errors + num_failures
     if num_problems == 0:
         info(ALL_TESTS_PASSED_MESSAGE % num_tests)
 
@@ -195,8 +195,9 @@ def __summarize_tests_full(
         message = PROBLEM_COUNT_MESSAGE % message_args
         warn(message)
 
-    for testcase_el in xunit_root.findall("testcase"):
-        __summarize_test_case(structured_data, testcase_el)
+    for testcase_el in test_results.xunit_testcase_elements:
+        structured_data_tests = test_results.structured_data_tests
+        __summarize_test_case(structured_data_tests, testcase_el)
 
 
 def __summarize_test_case(structured_data, testcase_el, **kwds):
@@ -279,8 +280,12 @@ def __structured_report_file(kwds, config):
         structured_data_supported = False
 
     structured_report_file = None
-    if structured_data_supported:
+    structured_report_file = kwds.get("test_output_json", None)
+    if structured_report_file is None and structured_data_supported:
         conf_dir = config.config_directory
-        structured_report_file = os.path.join(conf_dir, "structured_data.xml")
+        structured_report_file = os.path.join(conf_dir, "structured_data.json")
+    elif structured_report_file is not None and not structured_data_supported:
+        warn(NO_JSON_MESSAGE)
+        structured_report_file = None
 
     return structured_report_file

planemo/galaxy_test.py

@@ -0,0 +1,59 @@
+""" Utilities for reasoning about Galaxy test results.
+"""
+import json
+import xml.etree.ElementTree as ET
+
+
+class GalaxyTestResults(object):
+    """ Class that combine the test-centric xunit output
+    with the Galaxy centric structured data output - and
+    abstracts away the difference (someday).
+    """
+
+    def __init__(self, output_json_path, output_xml_path):
+        try:
+            output_json_f = open(output_json_path, "r")
+            structured_data = json.load(output_json_f)
+            structured_data_tests = structured_data["tests"]
+        except Exception:
+            # Older Galaxy's will not support this option.
+            structured_data = {}
+            structured_data_tests = {}
+        self.structured_data = structured_data
+        self.structured_data_tests = structured_data_tests
+        self.xunit_tree = ET.parse(output_xml_path)
+        self.__merge_xunit()
+        try:
+            json.dump(self.structured_data, open(output_json_path, "w"))
+        except Exception:
+            pass
+
+    @property
+    def _xunit_root(self):
+        return self.xunit_tree.getroot()
+
+    def __merge_xunit(self):
+        xunit_attrib = self._xunit_root.attrib
+        num_tests = int(xunit_attrib.get("tests", 0))
+        num_failures = int(xunit_attrib.get("failures", 0))
+        num_errors = int(xunit_attrib.get("errors", 0))
+        num_skips = int(xunit_attrib.get("skips", 0))
+        summary = dict(
+            num_tests=num_tests,
+            num_failures=num_failures,
+            num_errors=num_errors,
+            num_skips=num_skips,
+        )
+
+        self.structured_data["summary"] = summary
+        self.num_tests = num_tests
+        self.num_problems = num_skips + num_errors + num_failures
+
+    @property
+    def all_tests_passed(self):
+        return self.num_problems == 0
+
+    @property
+    def xunit_testcase_elements(self):
+        for testcase_el in self._xunit_root.findall("testcase"):
+            yield testcase_el