Add test_report command for rebuilding reports from structured JSON.

Refactor Galaxy testing abstractions toward better exception handling (print more, fail harder, try more - if one report type fails try the others but still throw an exception at the end) and toward greater reuse without being a full test context (i.e. try to be able to do more from just json).

planemo/commands/cmd_share_test.py

@@ -3,6 +3,7 @@
 import click
 
 from planemo.cli import pass_context
+from planemo import options
 from planemo.io import info
 from planemo import github_util
 
@@ -11,20 +12,9 @@
     "?test_data_url=%s"
 )
 
-target_path = click.Path(
-    file_okay=True,
-    dir_okay=False,
-    resolve_path=True,
-)
-
 
 @click.command("share_test")
-@click.argument(
-    'path',
-    metavar="FILE_PATH",
-    type=target_path,
-    default="tool_test_output.json",
-)
+@options.tool_test_json()
 @pass_context
 def cli(ctx, path, **kwds):
     """Publish JSON test results to Github Gist and produce sharable URL.

planemo/commands/cmd_test.py

@@ -24,7 +24,7 @@
 )
 @options.galaxy_target_options()
 @options.galaxy_config_options()
-@options.test_options()
+@options.test_report_options()
 @pass_context
 def cli(ctx, paths, **kwds):
     """Run the tests in the specified tool tests in a Galaxy instance.

planemo/commands/cmd_test_reports.py

@@ -0,0 +1,24 @@
+import os
+
+import click
+
+from planemo.cli import pass_context
+from planemo import io
+from planemo import options
+from planemo.galaxy_test import StructuredData, handle_test_reports
+
+
+@click.command('test_reports')
+@options.tool_test_json()
+@options.test_report_options()
+@pass_context
+def cli(ctx, path, **kwds):
+    """Generate various tool test reports (HTML, text, markdown) from
+    structure output from tests (tool_test_output.json).
+    """
+    if not os.path.exists(path):
+        io.error("Failed to tool test json file at %s" % path)
+        return 1
+
+    test_data = StructuredData(path)
+    handle_test_reports(ctx, test_data, **kwds)

planemo/galaxy_test/__init__.py

@@ -1,3 +1,6 @@
 from .actions import run_in_config
+from .structures import StructuredData
+from .actions import handle_test_reports
 
-__all__ = ["run_in_config"]
+
+__all__ = ["run_in_config", "StructuredData", "handle_test_reports"]

planemo/galaxy_test/actions.py

@@ -83,27 +83,8 @@ def run_in_config(ctx, config, **kwds):
         return_code,
     )
 
-    try:
-        test_data = test_results.structured_data
-
-        if 'test_output' in kwds:
-            output_path = kwds['test_output']
-            if output_path is not None:
-                with open(output_path, 'w') as handle:
-                    handle.write(build_report.build_report(test_data))
-
-        for kw_name in ('markdown', 'text'):
-            if 'test_output_%s' % kw_name in kwds:
-                output_path = kwds['test_output_%s' % kw_name]
-                if output_path is None:
-                    continue
-
-                with open(output_path, 'w') as handle:
-                    handle.write(build_report.build_report(test_data, report_type=kw_name))
-
-    except Exception:
-        ctx.vlog("Problem producing test output.", exception=True)
-
+    test_data = test_results.structured_data
+    handle_test_reports(ctx, test_data, **kwds)
     __handle_summary(
         test_results,
         **kwds
@@ -112,6 +93,54 @@ def run_in_config(ctx, config, **kwds):
     return return_code
 
 
+def handle_test_reports(ctx, test_data, **kwds):
+    exceptions = []
+    for report_type in ["html", "markdown", "text"]:
+        try:
+            _handle_test_output_file(
+                ctx, report_type, test_data, **kwds
+            )
+        except Exception as e:
+            exceptions.append(e)
+            continue
+
+    if len(exceptions) > 0:
+        raise exceptions[0]
+
+
+def _handle_test_output_file(ctx, report_type, test_data, **kwds):
+    kwd_name = "test_output"
+    if report_type != "html":
+        kwd_name = "test_output_%s" % report_type
+
+    path = kwds.get(kwd_name, None)
+    if path is None:
+        message = "No file specified for %s, skipping test output." % kwd_name
+        ctx.vlog(message)
+        return
+
+    try:
+        contents = build_report.build_report(
+            report_type, report_type=report_type
+        )
+    except Exception:
+        message = "Problem producing report file %s for %s" % (
+            path, kwd_name
+        )
+        ctx.vlog(message, exception=True)
+        raise
+
+    try:
+        with open(path, 'w') as handle:
+            handle.write(contents)
+    except Exception:
+        message = "Problem writing output file %s for %s" % (
+            kwd_name, path
+        )
+        ctx.vlog(message, exception=True)
+        raise
+
+
 def __handle_summary(
     test_results,
     **kwds

planemo/galaxy_test/structures.py

@@ -76,12 +76,15 @@ def __init__(self, json_path):
         self.structured_data_by_id = structured_data_by_id
         self.has_details = "summary" in structured_data
         if self.has_details:
-            self._read_summary()
+            self.read_summary()
 
     def update(self):
         with open(self.json_path, "w") as out_f:
             json.dump(self.structured_data, out_f)
 
+    def set_exit_code(self, exit_code):
+        self.structured_data["exit_code"] = exit_code
+
     def merge_xunit(self, xunit_root):
         self.has_details = True
         xunit_attrib = xunit_root.attrib
@@ -97,8 +100,6 @@ def merge_xunit(self, xunit_root):
         )
 
         self.structured_data["summary"] = summary
-        self.num_tests = num_tests
-        self.num_problems = num_skips + num_errors + num_failures
 
         for testcase_el in xunit_t_elements_from_root(xunit_root):
             test = case_id(testcase_el)
@@ -118,9 +119,17 @@ def merge_xunit(self, xunit_root):
                 status = "success"
             test_data["status"] = status
 
-    def _read_summary(self):
-        # TODO: read relevant data out of summary object.
-        pass
+    def read_summary(self):
+        summary = self.structured_data["summary"]
+        num_tests = summary["num_tests"]
+        num_failures = summary["num_failures"]
+        num_skips = summary["num_skips"]
+        num_errors = summary["num_errors"]
+
+        self.num_tests = num_tests
+        self.num_problems = num_skips + num_errors + num_failures
+
+        self.exit_code = summary["exit_code"]
 
     @property
     def failed_ids(self):
@@ -159,8 +168,14 @@ def __init__(
             sd.merge_xunit(self._xunit_root)
         else:
             self.xunit_tree = ET.fromstring("<testsuite />")
+        self.sd.set_exit_code(exit_code)
+        self.sd.read_summary()
         self.sd.update()
 
+    @property
+    def exit_code(self):
+        return self.sd.exit_code
+
     @property
     def has_details(self):
         return self.sd.has_details

planemo/options.py

@@ -616,14 +616,22 @@ def recursive_option(help="Recursively perform command for subdirectories."):
     )
 
 
-def test_options():
+def tool_test_json():
+    target_path = click.Path(
+        file_okay=True,
+        dir_okay=False,
+        resolve_path=True,
+    )
+    return click.argument(
+        'path',
+        metavar="FILE_PATH",
+        type=target_path,
+        default="tool_test_output.json",
+    )
+
+
+def test_report_options():
     return _compose(
-        click.option(
-            "--update_test_data",
-            is_flag=True,
-            help="Update test-data directory with job outputs (normally"
-                 " written to directory --job_output_files if specified.)"
-        ),
         click.option(
             "--test_output",
             type=click.Path(file_okay=True, resolve_path=True),
@@ -648,6 +656,18 @@ def test_options():
                   "computers)"),
             default=None,
         ),
+    )
+
+
+def test_options():
+    return _compose(
+        click.option(
+            "--update_test_data",
+            is_flag=True,
+            help="Update test-data directory with job outputs (normally"
+                 " written to directory --job_output_files if specified.)"
+        ),
+        test_report_options(),
         click.option(
             "--test_output_xunit",
             type=click.Path(file_okay=True, resolve_path=True),