Permalink
Browse files

Add test_report command for rebuilding reports from structured JSON.

Refactor Galaxy testing abstractions toward better exception handling (print more, fail harder, try more - if one report type fails try the others but still throw an exception at the end) and toward greater reuse without being a full test context (i.e. try to be able to do more from just json).
  • Loading branch information...
jmchilton committed Nov 23, 2015
1 parent 7572e99 commit 99ee51a1b800bc2bdbf4e1eee3d8579417185631
@@ -3,6 +3,7 @@
import click

from planemo.cli import pass_context
from planemo import options
from planemo.io import info
from planemo import github_util

@@ -11,20 +12,9 @@
"?test_data_url=%s"
)

target_path = click.Path(
file_okay=True,
dir_okay=False,
resolve_path=True,
)


@click.command("share_test")
@click.argument(
'path',
metavar="FILE_PATH",
type=target_path,
default="tool_test_output.json",
)
@options.tool_test_json()
@pass_context
def cli(ctx, path, **kwds):
"""Publish JSON test results to Github Gist and produce sharable URL.
@@ -24,7 +24,7 @@
)
@options.galaxy_target_options()
@options.galaxy_config_options()
@options.test_options()
@options.test_report_options()
@pass_context
def cli(ctx, paths, **kwds):
"""Run the tests in the specified tool tests in a Galaxy instance.
@@ -0,0 +1,24 @@
import os

import click

from planemo.cli import pass_context
from planemo import io
from planemo import options
from planemo.galaxy_test import StructuredData, handle_test_reports


@click.command('test_reports')
@options.tool_test_json()
@options.test_report_options()
@pass_context
def cli(ctx, path, **kwds):
"""Generate various tool test reports (HTML, text, markdown) from
structure output from tests (tool_test_output.json).
"""
if not os.path.exists(path):
io.error("Failed to tool test json file at %s" % path)
return 1

test_data = StructuredData(path)
handle_test_reports(ctx, test_data, **kwds)
@@ -1,3 +1,6 @@
from .actions import run_in_config
from .structures import StructuredData
from .actions import handle_test_reports

__all__ = ["run_in_config"]

__all__ = ["run_in_config", "StructuredData", "handle_test_reports"]
@@ -83,27 +83,8 @@ def run_in_config(ctx, config, **kwds):
return_code,
)

try:
test_data = test_results.structured_data

if 'test_output' in kwds:
output_path = kwds['test_output']
if output_path is not None:
with open(output_path, 'w') as handle:
handle.write(build_report.build_report(test_data))

for kw_name in ('markdown', 'text'):
if 'test_output_%s' % kw_name in kwds:
output_path = kwds['test_output_%s' % kw_name]
if output_path is None:
continue

with open(output_path, 'w') as handle:
handle.write(build_report.build_report(test_data, report_type=kw_name))

except Exception:
ctx.vlog("Problem producing test output.", exception=True)

test_data = test_results.structured_data
handle_test_reports(ctx, test_data, **kwds)
__handle_summary(
test_results,
**kwds
@@ -112,6 +93,54 @@ def run_in_config(ctx, config, **kwds):
return return_code


def handle_test_reports(ctx, test_data, **kwds):
exceptions = []
for report_type in ["html", "markdown", "text"]:
try:
_handle_test_output_file(
ctx, report_type, test_data, **kwds
)
except Exception as e:
exceptions.append(e)
continue

if len(exceptions) > 0:
raise exceptions[0]


def _handle_test_output_file(ctx, report_type, test_data, **kwds):
kwd_name = "test_output"
if report_type != "html":
kwd_name = "test_output_%s" % report_type

path = kwds.get(kwd_name, None)
if path is None:
message = "No file specified for %s, skipping test output." % kwd_name
ctx.vlog(message)
return

try:
contents = build_report.build_report(
report_type, report_type=report_type
)
except Exception:
message = "Problem producing report file %s for %s" % (
path, kwd_name
)
ctx.vlog(message, exception=True)
raise

try:
with open(path, 'w') as handle:
handle.write(contents)
except Exception:
message = "Problem writing output file %s for %s" % (
kwd_name, path
)
ctx.vlog(message, exception=True)
raise


def __handle_summary(
test_results,
**kwds
@@ -76,12 +76,15 @@ def __init__(self, json_path):
self.structured_data_by_id = structured_data_by_id
self.has_details = "summary" in structured_data
if self.has_details:
self._read_summary()
self.read_summary()

def update(self):
with open(self.json_path, "w") as out_f:
json.dump(self.structured_data, out_f)

def set_exit_code(self, exit_code):
self.structured_data["exit_code"] = exit_code

def merge_xunit(self, xunit_root):
self.has_details = True
xunit_attrib = xunit_root.attrib
@@ -97,8 +100,6 @@ def merge_xunit(self, xunit_root):
)

self.structured_data["summary"] = summary
self.num_tests = num_tests
self.num_problems = num_skips + num_errors + num_failures

for testcase_el in xunit_t_elements_from_root(xunit_root):
test = case_id(testcase_el)
@@ -118,9 +119,17 @@ def merge_xunit(self, xunit_root):
status = "success"
test_data["status"] = status

def _read_summary(self):
# TODO: read relevant data out of summary object.
pass
def read_summary(self):
summary = self.structured_data["summary"]
num_tests = summary["num_tests"]
num_failures = summary["num_failures"]
num_skips = summary["num_skips"]
num_errors = summary["num_errors"]

self.num_tests = num_tests
self.num_problems = num_skips + num_errors + num_failures

self.exit_code = summary["exit_code"]

@property
def failed_ids(self):
@@ -159,8 +168,14 @@ def __init__(
sd.merge_xunit(self._xunit_root)
else:
self.xunit_tree = ET.fromstring("<testsuite />")
self.sd.set_exit_code(exit_code)
self.sd.read_summary()
self.sd.update()

@property
def exit_code(self):

This comment has been minimized.

@nsoranzo

nsoranzo Nov 23, 2015

Member

exit_code is already an attribute of GalaxyTestResults class, defined in the __init__() method. Now instantiating an object of this class fails with AttributeError: can't set attribute.

return self.sd.exit_code

@property
def has_details(self):
return self.sd.has_details
@@ -616,14 +616,22 @@ def recursive_option(help="Recursively perform command for subdirectories."):
)


def test_options():
def tool_test_json():
target_path = click.Path(
file_okay=True,
dir_okay=False,
resolve_path=True,
)
return click.argument(
'path',
metavar="FILE_PATH",
type=target_path,
default="tool_test_output.json",
)


def test_report_options():
return _compose(
click.option(
"--update_test_data",
is_flag=True,
help="Update test-data directory with job outputs (normally"
" written to directory --job_output_files if specified.)"
),
click.option(
"--test_output",
type=click.Path(file_okay=True, resolve_path=True),
@@ -648,6 +656,18 @@ def test_options():
"computers)"),
default=None,
),
)


def test_options():
return _compose(
click.option(
"--update_test_data",
is_flag=True,
help="Update test-data directory with job outputs (normally"
" written to directory --job_output_files if specified.)"
),
test_report_options(),
click.option(
"--test_output_xunit",
type=click.Path(file_okay=True, resolve_path=True),
Oops, something went wrong.

0 comments on commit 99ee51a

Please sign in to comment.