Implement report templates

planemo/galaxy_test/actions.py

@@ -81,8 +81,16 @@ def run_in_config(ctx, config, **kwds):
 
     try:
         test_data = test_results.structured_data
-        new_report = build_report.build_report(test_data)
-        open(test_results.output_html_path, "w").write(new_report)
+
+        if 'test_output' in kwds:
+            with open(kwds['test_output'], 'w') as handle:
+                handle.write(build_report.build_report(test_data))
+
+        for kw_name in ('markdown', 'text'):
+            if 'test_output_%s' % kw_name in kwds:
+                with open(kwds['test_output_%s' % kw_name], 'w') as handle:
+                    handle.write(build_report.build_report(test_data, report_type=kw_name))
+
     except Exception:
         pass
 

planemo/options.py

@@ -560,6 +560,21 @@ def test_options():
                   "tool_test_output.html."),
             default=None,
         ),
+        click.option(
+            "--test_output_text",
+            type=click.Path(file_okay=True, resolve_path=True),
+            callback=get_default_callback(None),
+            help=("Output test report (Basic text - for display in CI)"),
+            default=None,
+        ),
+        click.option(
+            "--test_output_markdown",
+            type=click.Path(file_okay=True, resolve_path=True),
+            callback=get_default_callback(None),
+            help=("Output test report (Markdown style - for humans & "
+                  "computers)"),
+            default=None,
+        ),
         click.option(
             "--test_output_xunit",
             type=click.Path(file_okay=True, resolve_path=True),

planemo/reports/build_report.py

@@ -1,35 +1,32 @@
 import json
-from string import Template
 from pkg_resources import resource_string
-
+from jinja2 import Environment, PackageLoader
+env = Environment(loader=PackageLoader('planemo', 'reports'))
 TITLE = "Tool Test Results (powered by Planemo)"
-LINKS_HTML = """
-    <li><a href="https://galaxyproject.org">Galaxy</a></li>
-    <li><a href="https://planemo.readthedocs.org">Planemo</a></li>
-"""
 
 
-def build_report(structured_data, **kwds):
+def build_report(structured_data, report_type="html", **kwds):
     """ Use report_template.html to build HTML page for report.
     """
-    custom_style = __style("custom.css")
-    custom_script = __script("custom")
-    bootstrap_style = __style("bootstrap.min.css")
-    jquery_script = __script("jquery.min")
-    bootstrap_script = __script("bootstrap.min")
-
     environment = dict(
-        custom_style=custom_style,
-        custom_script=custom_script,
-        bootstrap_style=bootstrap_style,
-        jquery_script=jquery_script,
-        bootstrap_script=bootstrap_script,
         title=TITLE,
-        links=LINKS_HTML,
-        json_test_data=json.dumps(structured_data),
+        raw_data=structured_data,
     )
-    template = Template(__load_resource("report_template.html"))
-    return template.safe_substitute(environment)
+
+    if report_type == 'html':
+        # The HTML report format needs a lot of extra, custom data.
+        # IMO, this seems to suggest it should be embedded.
+        environment.update({
+            'custom_style': __style("custom.css"),
+            'custom_script': __script("custom"),
+            'bootstrap_style': __style("bootstrap.min.css"),
+            'jquery_script': __script("jquery.min"),
+            'bootstrap_script': __script("bootstrap.min"),
+            'json_test_data': json.dumps(structured_data),
+        })
+
+    template = env.get_template('report_%s.tpl' % report_type)
+    return template.render(**environment)
 
 
 def __style(filename):

planemo/reports/report_template.html → planemo/reports/report_html.tpl

@@ -4,11 +4,11 @@
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>${title}</title>
+    <title>{{ title }}</title>
 
     <!-- Bootstrap -->
-    ${bootstrap_style}
-    ${custom_style}
+    {{ bootstrap_style }}
+    {{ custom_style }}
 
     <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
     <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
@@ -30,11 +30,12 @@
             <span class="icon-bar"></span>
             <span class="icon-bar"></span>
           </button>
-          <a class="navbar-brand" href="#">${title}</a>
+          <a class="navbar-brand" href="#">{{ title }}</a>
         </div>
         <div id="navbar" class="navbar-collapse collapse">
           <ul class="nav navbar-nav navbar-right">
-            ${links}
+            <li><a href="https://galaxyproject.org">Galaxy</a></li>
+            <li><a href="https://planemo.readthedocs.org">Planemo</a></li>
           </ul>
           <div class="navbar-form navbar-right">
           </div>
@@ -63,9 +64,9 @@ <h2 id="tests">Tests</h2>
       </div>
     </div>
 
-    ${jquery_script}
-    ${bootstrap_script}
-    ${custom_script}
+    {{ jquery_script }}
+    {{ bootstrap_script }}
+    {{ custom_script }}
     <script>
       var testDataUrl = getUrlParameter("test_data_url");
       if(testDataUrl) {
@@ -77,7 +78,7 @@ <h2 id="tests">Tests</h2>
         .success(function(content) { renderTestResults( $.parseJSON(content) ); })
         .failure(function() { alert("Failed to load test data.")} );
       } else {
-        var test_data = ${json_test_data};
+        var test_data = {{ json_test_data }};
         renderTestResults(test_data);
       }
     </script>

planemo/reports/report_markdown.tpl

@@ -0,0 +1,55 @@
+# {{ title }}
+
+## Executive Summary
+
+| Test State | Count |
+| ---------- | ----- |
+| Total      | {{ raw_data.summary.num_tests | default(0)  }} |
+| Passed     | {{ raw_data.summary.num_tests - raw_data.summary.num_errors - raw_data.summary.num_failures - raw_data.summary.num_skips | default(0)  }} |
+| Error      | {{ raw_data.summary.num_errror | default(0) }} |
+| Failure    | {{ raw_data.summary.num_failure | default(0) }} |
+| Skipped    | {{ raw_data.summary.num_skipped | default(0) }} |
+
+
+## Detailed Results
+{% for test in raw_data.tests %}
+### {{ test.id }}
+{% if test.data.status == 'success' %}
+Job Passed
+{% else %}
+Job Error! (State: {{ test.data.status }})
+
+Command Line:
+
+```console
+{{ test.data.job.command_line}}
+```
+
+exited with code {{ test.data.job.exit_code }}.
+
+#### Problems
+
+{% for problem in test.data.output_problems %}
+```console
+{{problem}}
+```
+{% endfor %}
+
+{% if test.data.job.stdout %}
+#### `stderr`
+
+```console
+{{ test.data.job.stderr}}
+```
+
+{% endif %}
+{% if test.data.job.stdout %}
+#### `stdout`
+
+```console
+{{ test.data.job.stdout}}
+```
+
+{% endif %}
+{% endif %}
+{% endfor %}

planemo/reports/report_text.tpl

@@ -0,0 +1,15 @@
+{% set bad = raw_data.summary.num_errors + raw_data.summary.num_failures + raw_data.summary.num_skips -%}
+{% if bad == 0 -%}
+All {{ raw_data.summary.num_tests }} test(s) executed passed.
+{%- else -%}
+There were problems with {{ bad }} out of {{raw_data.summary.num_tests}} test(s) executed.
+{%- endif %}
+
+{% for test in raw_data.tests -%}
+{{ test.id | replace('functional.test_toolbox.TestForTool_', '') }}: {{ test.data.status }}
+{% if test.data.output_problems -%}
+{% for problem in test.data.output_problems -%}
+{{problem | indent(2, True) }}
+{% endfor %}
+{% endif %}
+{%- endfor %}