Permalink
Browse files

Overhaul testing.

 - Refactor to cleanup test_planemo, introduce higher-level concepts in new CliTestCase.
 - Add integration test for tool initialization and linting.
 - Bug fixes for tool linting overcovered with these commands.
 - Add integration test for initializing demo project and testing (cat.xml).
 - Add DOI to tool init (so we can have a lintable passing tool from init to test).
 - Add test for --help at root command.
 - Add test for --version at root command.
 - Update dev-requirements to include nose and coverage.
 - Add coveralls to travis testing.
  • Loading branch information...
jmchilton committed Mar 9, 2015
1 parent 51896f5 commit 0bd4ff07210af8156de8457637b7855abdc6ac7f
@@ -11,8 +11,12 @@ python:

install:
- if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then pip install unittest2; fi
- pip install coveralls --use-mirrors # Required for coveralls reporting.
- pip install -r requirements.txt
- pip install pyflakes flake8 readme
- pip install pyflakes flake8 readme coverage

after_success:
- coveralls

# command to run tests, e.g. python setup.py test
script: make lint && make test && if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then true; else make lint-readme; fi
@@ -1,3 +1,7 @@
# For testing
nose
coverage

# For dev
sphinx

@@ -152,6 +152,15 @@
prompt=False,
help="Add a Docker image identifier for this tool."
)
@click.option(
"--doi",
type=click.STRING,
default=None,
multiple=True,
prompt=False,
help=("Supply a DOI (http://www.doi.org/) easing citation of the tool "
"for Galxy users (e.g. 10.1101/014043).")
)
@click.option(
"--test_case",
is_flag=True,
@@ -62,6 +62,13 @@
TODO: Fill in help.
{%- endif %}
]]></help>
{%- if doi %}
<citations>
{%- for single_doi in doi %}
<citation type="doi">{{ single_doi }}</citation>
{%- endfor %}
</citations>
{%- endif %}
</tool>
"""

@@ -17,10 +17,11 @@ def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN):
lint_context.lint(module, name, value, tool_xml)
found_warns = lint_context.found_warns
found_errors = lint_context.found_errors
if level == LEVEL_WARN and (found_warns or found_errors):
return False
if fail_level == LEVEL_WARN:
lint_fail = (found_warns or found_errors)
else:
return found_errors
lint_fail = found_errors
return not lint_fail


class LintContext(object):
@@ -0,0 +1,62 @@

from .test_utils import CliTestCase


INIT_COMMAND = [
"tool_init", "--force",
"--id", "seqtk_seq",
"--name", "Convert to FASTA (seqtk)",
"--requirement", "seqtk@1.0-r68",
"--example_command", "seqtk seq -a 2.fastq > 2.fasta",
"--example_input", "2.fastq",
"--example_output", "2.fasta",
"--test_case",
"--help_text", "The help text.",
"--doi", "10.1101/014043"
]


class BuildAndLintTestCase(CliTestCase):

def test_build_and_lint(self):
with self._isolate():
self._check_exit_code(_init_command())
self._check_lint(exit_code=0)

def test_lint_fails_if_no_help(self):
with self._isolate():
self._check_exit_code(_init_command(help_text=False))
self._check_lint(exit_code=1)

def test_lint_fails_if_no_test(self):
with self._isolate():
self._check_exit_code(_init_command(test_case=False))
self._check_lint(exit_code=1)

def test_lint_fails_if_no_doi(self):
with self._isolate():
self._check_exit_code(_init_command(doi=False))
self._check_lint(exit_code=1)

def _check_lint(self, exit_code=0):
lint_cmd = ["lint", "--fail_level", "warn", "seqtk_seq.xml"]
self._check_exit_code(lint_cmd, exit_code=exit_code)


def _init_command(test_case=True, help_text=True, doi=True):
command = [
"tool_init", "--force",
"--id", "seqtk_seq",
"--name", "Convert to FASTA (seqtk)",
"--requirement", "seqtk@1.0-r68",
"--example_command", "seqtk seq -a 2.fastq > 2.fasta",
"--example_input", "2.fastq",
"--example_output", "2.fasta"
]
if test_case:
command.append("--test_case")
if help_text:
command.extend(["--help_text", "The help text."])
if doi:
command.extend(["--doi", "10.1101/014043"])
return command
@@ -0,0 +1,11 @@
from .test_utils import CliTestCase


class InitAndTestTestCase(CliTestCase):

def test_init_and_test(self):
with self._isolate():
init_cmd = ["project_init", "--template", "demo", "basic"]
self._check_exit_code(init_cmd)
test_cmd = ["test", "--install_galaxy", "basic/cat.xml"]
self._check_exit_code(test_cmd)
@@ -8,48 +8,50 @@
Tests for `planemo` module.
"""

from click.testing import CliRunner
from .test_utils import CliTestCase
from .test_utils import main

import unittest

from planemo import cli


# More information on testing click applications at following link.
# http://click.pocoo.org/3/testing/#basic-testing
class TestPlanemo(unittest.TestCase):
class TestPlanemo(CliTestCase):

def test_commands_have_help(self):
commands = cli.list_cmds()
runner = CliRunner()
commands = self._cli.list_cmds()
for command in commands:
planemo_cli = cli.planemo
result = runner.invoke(planemo_cli, [command, "--help"])
if result.exit_code != 0:
message = "Planemo command %s has invalid --help." % command
raise AssertionError(message)
self._check_exit_code([command, "--help"])

def test_responds_to_desired_commands(self):
commands = cli.list_cmds()
commands = self._cli.list_cmds()

def assert_responds_to(command):
assert command in commands, "No command %s" % command

assert_responds_to("docker_shell")
assert_responds_to("docker_build")
assert_responds_to("brew_init")
assert_responds_to("brew")
assert_responds_to("brew_env")
assert_responds_to("config_init")
assert_responds_to("create_gist")
assert_responds_to("docker_build")
assert_responds_to("docker_shell")
assert_responds_to("lint")
assert_responds_to("normalize")
assert_responds_to("project_init")
assert_responds_to("shed_upload")
assert_responds_to("serve")
assert_responds_to("shed_diff")
assert_responds_to("shed_download")
assert_responds_to("shed_upload")
assert_responds_to("syntax")
assert_responds_to("test")
assert_responds_to("tool_factory")
assert_responds_to("tool_init")
assert_responds_to("travis_before_install")
assert_responds_to("travis_init")

def test_planemo_version_command(self):
self._check_exit_code(["--version"])

def test_planemo_help_command(self):
self._check_exit_code(["--help"])


if __name__ == '__main__':
unittest.main()
main()
@@ -0,0 +1,52 @@
""" Provide abstractions over click testing of the
app and unittest.
"""
from click.testing import CliRunner

from unittest import (
TestCase,
main,
)
from planemo import cli

EXIT_CODE_MESSAGE = ("Planemo command [%s] resulted in unexpected exit code "
"[%s], expected exit code [%s]]. Command output [%s]")


# More information on testing click applications at following link.
# http://click.pocoo.org/3/testing/#basic-testing
class CliTestCase(TestCase):

def setUp(self): # noqa
self._runner = CliRunner()

@property
def _cli(self):
return cli

def _isolate(self):
return self._runner.isolated_filesystem()

def _invoke(self, command_list):
planemo_cli = self._cli.planemo
return self._runner.invoke(planemo_cli, command_list)

def _check_exit_code(self, command_list, exit_code=0):
expected_exit_code = exit_code
result = self._invoke(command_list)
result_exit_code = result.exit_code
if result_exit_code != expected_exit_code:
message = EXIT_CODE_MESSAGE % (
" ".join(command_list),
result_exit_code,
expected_exit_code,
result.output,
)
raise AssertionError(message)


__all__ = [
"TestCase",
"CliTestCase",
"main",
]

0 comments on commit 0bd4ff0

Please sign in to comment.