Permalink
Browse files

Make planemo test a little less error prone.

 - Insist that Galaxy produce json and xunit test data. Previously planemo would try to work through without these files and just issue warnings - but the state could be inconsistent.
 - Some initial unit tests for planemo test - mocking out the actual running of Galaxy (so we can simulate problems generating these files).
  • Loading branch information...
jmchilton committed May 11, 2016
1 parent 52dc27e commit f3c6917a400e9c30e036809696ef973d069d0f5c
Showing with 477 additions and 30 deletions.
  1. +32 −28 planemo/galaxy/test/actions.py
  2. +370 −0 tests/data/tt_success.html
  3. +1 −0 tests/data/tt_success.json
  4. +1 −0 tests/data/tt_success.xml
  5. +73 −2 tests/test_galaxy_test.py
@@ -4,7 +4,7 @@
import click

from . import structures as test_structures
from planemo.io import info, warn, shell_join
from planemo.io import error, info, warn, shell_join
from planemo.galaxy.run import (
run_galaxy_command,
setup_venv,
@@ -14,12 +14,12 @@

from galaxy.tools.deps.commands import shell

XUNIT_UPGRADE_MESSAGE = ("This version of Galaxy does not support xUnit - "
"please update to newest development brach.")
NO_XUNIT_MESSAGE = ("Cannot locate xUnit report option for tests - update "
"Galaxy for more detailed breakdown.")
NO_JSON_MESSAGE = ("Cannot locate json report option for tests - update "
"Galaxy for more detailed breakdown.")
NO_XUNIT_REPORT_MESSAGE = ("Cannot locate xUnit report [%s] for tests - "
"required to build planemo report and summarize "
"tests.")
NO_JSON_REPORT_MESSAGE = ("Cannot locate JSON report [%s] for tests - "
"required to build planemo report and summarize "
"tests.")
NO_TESTS_MESSAGE = "No tests were executed - see Galaxy output for details."
ALL_TESTS_PASSED_MESSAGE = "All %d test(s) executed passed."
PROBLEM_COUNT_MESSAGE = ("There were problems with %d test(s) - out of %d "
@@ -29,7 +29,7 @@
GENERIC_TESTS_PASSED_MESSAGE = "No failing tests encountered."


def run_in_config(ctx, config, **kwds):
def run_in_config(ctx, config, run=run_galaxy_command, **kwds):
"""Run Galaxy tests with the run_tests.sh command.
The specified `config` object describes the context for tool
@@ -42,7 +42,7 @@ def run_in_config(ctx, config, **kwds):
if job_output_files is None:
job_output_files = os.path.join(config_directory, "jobfiles")

xunit_supported, xunit_report_file = _xunit_state(kwds, config)
xunit_report_file = _xunit_state(kwds, config)
structured_report_file = _structured_report_file(kwds, config)

info("Testing using galaxy_root %s", config.galaxy_root)
@@ -75,7 +75,7 @@ def run_in_config(ctx, config, **kwds):
test_cmd,
)
action = "Testing tools"
return_code = run_galaxy_command(
return_code = run(
ctx,
cmd,
config.env,
@@ -85,9 +85,15 @@ def run_in_config(ctx, config, **kwds):
update_cp_args = (job_output_files, config.test_data_dir)
shell('cp -r "%s"/* "%s"' % update_cp_args)

if xunit_report_file and (not os.path.exists(xunit_report_file)):
warn(NO_XUNIT_MESSAGE)
xunit_report_file = None
if not os.path.exists(xunit_report_file):
message = NO_XUNIT_REPORT_MESSAGE % xunit_report_file
error(message)
raise Exception(message)

if not os.path.exists(structured_report_file):
message = NO_JSON_REPORT_MESSAGE % structured_report_file
error(message)
raise Exception(message)

test_results = test_structures.GalaxyTestResults(
structured_report_file,
@@ -243,33 +249,31 @@ def _print_command_line(structured_data, test_id):


def _xunit_state(kwds, config):
xunit_supported = True
if shell("grep -q xunit '%s'/run_tests.sh" % config.galaxy_root):
xunit_supported = False
# This has been supported in Galaxy for well over a year, just going to assume
# it from here on out.
# xunit_supported = True
# if shell("grep -q xunit '%s'/run_tests.sh" % config.galaxy_root):
# xunit_supported = False

xunit_report_file = kwds.get("test_output_xunit", None)
if xunit_report_file is None and xunit_supported:
if xunit_report_file is None:
xunit_report_file = os.path.join(config.config_directory, "xunit.xml")
elif xunit_report_file is not None and not xunit_supported:
warn(XUNIT_UPGRADE_MESSAGE)
xunit_report_file = None

return xunit_supported, xunit_report_file
return xunit_report_file


def _structured_report_file(kwds, config):
structured_data_supported = True
if shell("grep -q structured_data '%s'/run_tests.sh" % config.galaxy_root):
structured_data_supported = False
# This has been supported in Galaxy for well over a year, just going to assume
# it from here on out.
# structured_data_supported = True
# if shell("grep -q structured_data '%s'/run_tests.sh" % config.galaxy_root):
# structured_data_supported = False

structured_report_file = None
structured_report_file = kwds.get("test_output_json", None)
if structured_report_file is None and structured_data_supported:
if structured_report_file is None:
conf_dir = config.config_directory
structured_report_file = os.path.join(conf_dir, "structured_data.json")
elif structured_report_file is not None and not structured_data_supported:
warn(NO_JSON_MESSAGE)
structured_report_file = None

return structured_report_file

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -0,0 +1 @@
{"tests": [{"data": {"status": "success", "inputs": {"input1": {"src": "hda", "id": "2891970512fa2d5a"}}, "job": {"inputs": {"input1": {"src": "hda", "id": "2891970512fa2d5a", "uuid": "3b1f7b73-d96b-4ca8-be81-3aa7a24bac13"}}, "update_time": "2016-05-11T00:55:26.723036", "tool_id": "cat", "outputs": {"out_file1": {"src": "hda", "id": "5729865256bc2525", "uuid": "c7bf3ebb-1bed-45ce-b473-aefba37773ec"}}, "stdout": "", "exit_code": 0, "state": "ok", "create_time": "2016-05-11T00:55:23.253017", "params": {"chromInfo": "\"/home/john/workspace/galaxy/tool-data/shared/ucsc/chrom/hg17.len\"", "dbkey": "\"hg17\"", "queries": "[]"}, "stderr": "", "model_class": "Job", "id": "5729865256bc2525"}}, "id": "functional.test_toolbox.TestForTool_cat.test_tool_000000", "has_data": true}], "version": "0.1", "exit_code": 0, "summary": {"num_skips": 0, "num_errors": 0, "num_failures": 0, "num_tests": 1}}
@@ -0,0 +1 @@
<?xml version="1.0" encoding="UTF-8"?><testsuite name="nosetests" tests="1" errors="0" failures="0" skip="0"><testcase classname="functional.test_toolbox.TestForTool_cat" name="test_tool_000000" time="12.805"></testcase></testsuite>
@@ -1,23 +1,85 @@
""" Tests for the galaxy_test module :) """
"""Tests for the `planeo.galaxy.test` module."""

import os
import shutil

from .test_utils import TEST_DATA_DIR
from .test_utils import (
test_context,
TempDirectoryTestCase,
TEST_DATA_DIR,
)

from planemo.galaxy.test import structures
from planemo.galaxy.test.actions import passed
from planemo.galaxy.test.actions import run_in_config


nose_1_3_report = os.path.join(TEST_DATA_DIR, "xunit_nose_1_3.xml")
nose_0_11_report = os.path.join(TEST_DATA_DIR, "xunit_nose_0_11.xml")

xunit_report_with_failure = os.path.join(TEST_DATA_DIR, "xunit_failure.xml")


class RunInConfigTestCase(TempDirectoryTestCase):
"""Test cases for ``run_in_config``."""

def setUp(self):
"""Setup mock keywords, context, and Galaxy config for tests."""
super(RunInConfigTestCase, self).setUp()
td = self.temp_directory
self.ctx = test_context()
self.config = _MockConfig(td)
self.kwds = {
"test_output": os.path.join(td, "tests.html"),
"test_output_json": os.path.join(td, "tests.json"),
"test_output_xunit": os.path.join(td, "tests.xml"),
}

def test_normal_execution(self):
"""Test normal operation of run_in_config."""
def mock_galaxy_run(ctx_, command, env, action):
assert ctx_ is self.ctx
assert env["test_key"] == "test_value"
self._copy_good_artifacts(["xml", "html", "json"])
return 0

assert self._do_run(mock_galaxy_run) == 0

def test_failed_to_produce_xunit(self):
"""Test an exception is thrown if not XUnit report is produced."""
def mock_galaxy_run(ctx_, command, env, action):
self._copy_good_artifacts(["json", "html"])
return 0

with self.assertRaises(Exception):
self._do_run(mock_galaxy_run)

def test_failed_to_produce_json(self):
"""Test an exception is thrown if not XUnit report is produced."""
def mock_galaxy_run(ctx_, command, env, action):
self._copy_good_artifacts(["xml", "html"])
return 0

with self.assertRaises(Exception):
self._do_run(mock_galaxy_run)

def _copy_good_artifacts(self, extensions):
for extension in extensions:
source = os.path.join(TEST_DATA_DIR, "tt_success.%s" % extension)
destination = os.path.join(self.temp_directory, "tests.%s" % extension)
shutil.copy(source, destination)

def _do_run(self, mock_run_function):
return run_in_config(self.ctx, self.config, run=mock_run_function, **self.kwds)


def get_test_id_new():
"""Test ID parsing on newer nose dependency."""
_get_test_id(nose_1_3_report)


def get_test_id_old():
"""Test ID parsing on older nose dependency."""
_get_test_id(nose_0_11_report)


@@ -33,10 +95,19 @@ def _get_test_id(path):


def test_passed():
"""Test :func:`passed`."""
xml_tree = structures.parse_xunit_report(xunit_report_with_failure)
root = xml_tree.getroot()
good_testcase_el = structures.find_cases(root)[0]
assert passed(good_testcase_el)

bad_testcase_el = structures.find_cases(root)[1]
assert not passed(bad_testcase_el)


class _MockConfig(object):

def __init__(self, temp_directory):
self.config_directory = temp_directory
self.env = {"test_key": "test_value"}
self.galaxy_root = os.path.join(self.config_directory, "galaxy-dev")

0 comments on commit f3c6917

Please sign in to comment.