Prettier, more structured HTML reports for tests.

This version of the HTML report includes information pulled out in a structured way such command-line, job standard error and standard output, failure reasons, etc... and has a much more modern look and feel thanks to bootstrap.

Basically - this replaces the HTML report generated by Galaxy with one generated here in planemo using the structured data json stuff. It generates a single static HTML page with jquery and bootstrap embedded directly in the page as well as custom JavaScript code for parsing and displaying the structured data JSON.

Before and after screenshots at http://imgur.com/a/TZYJq.

MANIFEST.in

@@ -4,6 +4,10 @@ include HISTORY.rst
 include LICENSE
 include README.rst
 
+include planemo/reports/*css
+include planemo/reports/*js
+include planemo/reports/*html
+
 recursive-include tests *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]

planemo/commands/cmd_test.py

@@ -9,6 +9,8 @@
 from planemo import galaxy_config
 from planemo import galaxy_run
 from planemo import galaxy_test
+from planemo.reports import build_report
+
 
 from galaxy.tools.deps.commands import shell
 
@@ -144,6 +146,13 @@ def cli(ctx, path, **kwds):
             return_code,
         )
 
+        try:
+            test_data = test_results.structured_data
+            new_report = build_report.build_report(test_data)
+            open(test_results.output_html_path, "w").write(new_report)
+        except Exception:
+            pass
+
         __handle_summary(
             test_results,
             **kwds

planemo/reports/build_report.py

@@ -0,0 +1,48 @@
+import json
+from string import Template
+from pkg_resources import resource_string
+
+TITLE = "Tool Test Results (powered by Planemo)"
+LINKS_HTML = """
+    <li><a href="https://galaxyproject.org">Galaxy</a></li>
+    <li><a href="https://planemo.readthedocs.com">Planemo</a></li>
+"""
+
+
+def build_report(structured_data, **kwds):
+    """ Use report_template.html to build HTML page for report.
+    """
+    custom_style = __style("custom.css")
+    custom_script = __script("custom")
+    bootstrap_style = __style("bootstrap.min.css")
+    jquery_script = __script("jquery.min")
+    bootstrap_script = __script("bootstrap.min")
+
+    environment = dict(
+        custom_style=custom_style,
+        custom_script=custom_script,
+        bootstrap_style=bootstrap_style,
+        jquery_script=jquery_script,
+        bootstrap_script=bootstrap_script,
+        title=TITLE,
+        links=LINKS_HTML,
+        json_test_data=json.dumps(structured_data),
+    )
+    template = Template(__load_resource("report_template.html"))
+    return template.safe_substitute(environment)
+
+
+def __style(filename):
+    resource = __load_resource(filename)
+    return "<style>%s</style>" % resource
+
+
+def __script(short_name):
+    resource = __load_resource("%s.js" % short_name)
+    return "<script>%s</script>" % resource
+
+
+def __load_resource(name):
+    return resource_string(
+        __name__, name
+    ).decode('UTF-8')

planemo/reports/custom.css

@@ -0,0 +1,106 @@
+/* bootstrap custom style stuff - taken from demo template. */
+/*
+ * Base structure
+ */
+
+/* Move down content because we have a fixed navbar that is 50px tall */
+body {
+  padding-top: 50px;
+}
+
+
+/*
+ * Global add-ons
+ */
+
+.sub-header {
+  padding-bottom: 10px;
+  border-bottom: 1px solid #eee;
+}
+
+/*
+ * Top navigation
+ * Hide default border to remove 1px line.
+ */
+.navbar-fixed-top {
+  border: 0;
+}
+
+/*
+ * Sidebar
+ */
+
+/* Hide for mobile, show later */
+.sidebar {
+  display: none;
+}
+@media (min-width: 768px) {
+  .sidebar {
+    position: fixed;
+    top: 51px;
+    bottom: 0;
+    left: 0;
+    z-index: 1000;
+    display: block;
+    padding: 20px;
+    overflow-x: hidden;
+    overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
+    background-color: #f5f5f5;
+    border-right: 1px solid #eee;
+  }
+}
+
+/* Sidebar navigation */
+.nav-sidebar {
+  margin-right: -21px; /* 20px padding + 1px border */
+  margin-bottom: 20px;
+  margin-left: -20px;
+}
+.nav-sidebar > li > a {
+  padding-right: 20px;
+  padding-left: 20px;
+}
+.nav-sidebar > .active > a,
+.nav-sidebar > .active > a:hover,
+.nav-sidebar > .active > a:focus {
+  color: #fff;
+  background-color: #428bca;
+}
+
+
+/*
+ * Main content
+ */
+
+.main {
+  padding: 20px;
+}
+@media (min-width: 768px) {
+  .main {
+    padding-right: 40px;
+    padding-left: 40px;
+  }
+}
+.main .page-header {
+  margin-top: 0;
+}
+
+
+/*
+ * Placeholder dashboard ideas
+ */
+
+.placeholders {
+  margin-bottom: 30px;
+  text-align: center;
+}
+.placeholders h4 {
+  margin-bottom: 0;
+}
+.placeholder {
+  margin-bottom: 20px;
+}
+.placeholder img {
+  display: inline-block;
+  border-radius: 50%;
+}

planemo/reports/custom.js

@@ -0,0 +1,132 @@
+
+var renderTestResults = function(testData) {
+	var summary = testData["summary"];
+	var numTests = summary["num_tests"];
+	var numProblems = summary["num_errors"] + summary["num_failures"] + summary["num_skips"];
+	var $overview = $("#overview-content");
+	var $progress = $(".progress");
+	if(numTests == 0) {
+		$overview.addClass("alert").addClass("alert-danger").text("No tests were executed.");
+		$progress.append($('<div class="progress-bar progress-bar-warning" role="progressbar" style="width: 100%" />'));
+	} else if(numProblems > 0) {
+		$overview.addClass("alert").addClass("alert-danger").text("There were problems with " + numProblems + " test(s) out of " + numTests + ".");
+		var problemPercent = (numProblems/(1.0 * numTests)) * 100.0;
+		var successPercent = 100.0 - problemPercent;
+		$progress.append($('<div class="progress-bar progress-bar-success" role="progressbar" style="width: ' + successPercent  +  '%" />'));
+		$progress.append($('<div class="progress-bar progress-bar-danger" role="progressbar" style="width: ' + problemPercent  +  '%" />'));
+	} else {
+		$overview.addClass("alert").addClass("alert-success").text("All " + numTests + " test(s) successfully executed.");
+		$progress.append($('<div class="progress-bar progress-bar-success" role="progressbar" style="width: 100%" />'));
+	}
+
+	var $sidebar = $("#nav-sidebar-tests");
+	for(var index in testData["tests"]) {
+		var test = testData["tests"][index];
+		var testResult = new TestResult(test);
+		var rawId = testResult.rawId;
+
+		var panelType = testResult.passed ? "panel-success" : "panel-danger";
+		var $panel = $('<div class="panel">');
+		$panel.addClass(panelType);
+
+		var $panelHeading = $('<div class="panel-heading">');
+		var $panelTitle = $('<div class="panel-title">');
+		var $a = $('<a class="collapsed" data-toggle="collapse">');
+		$a.attr("id", rawId);
+		$a.attr("data-target", "#collapse"  + index);
+		var testName = testResult.toolName + " (Test #" + (testResult.testIndex + 1) + ")"
+		$a.text(testName);
+		var $navLink = $('<a>').attr('href', '#' + rawId).text(testName)
+		if(!testResult.passed) {
+			$navLink.addClass("text-danger");
+		} else {
+			$navLink.addClass("text-success");
+		}
+		$sidebar.append($('<li>').append( $navLink ) );
+		$panelTitle.append($a)
+		$panelHeading.append($panelTitle);
+
+		var $panelBody = $('<div class="panel-body panel-collapse collapse" >');
+		$panelBody.attr("id", "collapse" + index);
+
+		var $status = $('<div>').text("status: " + testResult.status);
+		$panelBody.append($status);
+		if(testResult.problems.length > 0) {
+			var $problemsLabel = $('<div>').text("problems: ");
+			var $problemsDiv = $('<div style="margin-left:10px;">');
+			var $problemsUl = $('<ul>');
+			for(var problemIndex in testResult.problems) {
+				$problemsUl.append($('<li>').append($('<code>').text(testResult.problems[problemIndex])));
+			}
+			$problemsDiv.append($problemsUl);
+			$panelBody.append($problemsLabel).append($problemsDiv);
+		}
+		var $commandLabel = $('<div>command:</div>');
+		var $stdoutLabel = $('<div>job standard output:</div>');
+		var $stderrLabel = $('<div>job standard error:</div>');
+		var $command;
+		if(testResult.command !== null) {
+			$command = $('<pre class="pre-scrollable" style="margin-left:10px;">').text(testResult.command);
+		} else {
+			$command = $('<div class="alert alert-warning" style="margin-left:10px;">').text("No command recorded.");
+		}
+		var $stdout;
+		if(testResult.stdout !== null) {
+			$stdout = $('<pre class="pre-scrollable" style="margin-left:10px;">').text(testResult.stdout);
+		} else {
+			$stdout = $('<div class="alert alert-warning" style="margin-left:10px;">').text("No standard output recorded.");
+		}
+		var $stderr;
+		if(testResult.stderr !== null) {
+			$stderr = $('<pre class="pre-scrollable" style="margin-left:10px;">').text(testResult.stderr);
+		} else {
+			$stderr = $('<div class="alert alert-warning" style="margin-left:10px;">').text("No standard error recorded.");
+		}
+		$panelBody
+			.append($commandLabel)
+			.append($command)
+			.append($stdoutLabel)
+			.append($stdout)
+			.append($stderrLabel)
+			.append($stderr);
+		if(!testResult.passed) {
+			var $logLabel = $('<div>log:</div>');
+			var $log = $('<pre class="pre-scrollable" style="margin-left: 10px;">').text(testResult.problemLog);
+			$panelBody.append($logLabel).append($log);
+		}
+
+		$panel.append($panelHeading).append($panelBody);
+		$(".main").append($panel);
+	}
+}
+
+var TestResult = function(data) {
+	this.rawId = data["id"];
+
+	var testMethod = this.rawId.split("TestForTool_")[1];
+	var toolName = testMethod.split(".test_tool_")[0];
+	var testIndex = testMethod.split(".test_tool_")[1];
+	this.toolName = toolName;
+	this.testIndex = parseInt(testIndex);
+	console.log(data);
+	this.status = data["data"]["status"];
+	var job = data["data"]["job"];
+	if(job) {
+		this.stdout = data["data"]["job"]["stdout"];
+		this.stderr = data["data"]["job"]["stderr"];
+		this.command = data["data"]["job"]["command_line"];		
+	} else {
+		this.stdout = null;
+		this.stderr = null;
+		this.command = null;
+	}
+	this.problems = [];
+	var outputProblems = data["data"]["output_problems"] || [];
+	var executionProblem = data["data"]["execution_problem"];
+	this.problems.push.apply(this.problems, outputProblems);
+	if(executionProblem) {
+		this.problems.push(executionProblem);
+	}
+	this.problemLog = data["data"]["problem_log"];
+	this.passed = (this.status == "success");
+}

planemo/reports/report_template.html

@@ -0,0 +1,74 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>${title}</title>
+
+    <!-- Bootstrap -->
+    ${bootstrap_style}
+    ${custom_style}
+
+    <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+    <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+
+  </head>
+  <body>
+
+    <nav class="navbar navbar-inverse navbar-fixed-top" role="navigation">
+      <div class="container-fluid">
+        <div class="navbar-header">
+
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">${title}</a>
+        </div>
+        <div id="navbar" class="navbar-collapse collapse">
+          <ul class="nav navbar-nav navbar-right">
+            ${links}
+          </ul>
+          <div class="navbar-form navbar-right">
+          </div>
+        </div>
+      </div>
+    </nav>
+
+    <div class="container-fluid">
+      <div class="row">
+        <div class="col-sm-3 col-md-2 sidebar">
+          <ul class="nav nav-sidebar">
+            <li><a href="#overview" class="text-success"><strong>Overview</strong></a></li>
+          </ul>
+          <ul class="nav nav-sidebar" id="nav-sidebar-tests">
+          </ul>
+        </div>
+        <div class="col-sm-9 col-sm-offset-3 col-md-10 col-md-offset-2 main">
+          <!-- <h1 class="page-header">Tests</h1> -->
+          <h2 id="overview">Overview</h2>
+          <div id="overview-content"></div>
+          <div class="progress">
+          </div>
+          <h2 id="tests">Tests</h2>
+          <p>The remainder of this contains a description for each test executed to run these jobs.</p>
+        </div>
+      </div>
+    </div>
+
+    ${jquery_script}
+    ${bootstrap_script}
+    ${custom_script}
+    <script>
+      var test_data = ${json_test_data};
+      renderTestResults(test_data);
+    </script>
+  </body>
+</html>