Overhaul testing.

- Refactor to cleanup test_planemo, introduce higher-level concepts in new CliTestCase. - Add integration test for tool initialization and linting. - Bug fixes for tool linting overcovered with these commands. - Add integration test for initializing demo project and testing (cat.xml). - Add DOI to tool init (so we can have a lintable passing tool from init to test). - Add test for --help at root command. - Add test for --version at root command. - Update dev-requirements to include nose and coverage. - Add coveralls to travis testing.
galaxyproject · Mar 9, 2015 · 0bd4ff07210af8156de8457637b7855abdc6ac7f · 0bd4ff0
1 parent 51896f5
commit 0bd4ff07210af8156de8457637b7855abdc6ac7f
diff --git a/.travis.yml b/.travis.yml
@@ -11,8 +11,12 @@ python:
 install:
  - if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then pip install unittest2; fi
+  
+  - pip install coveralls --use-mirrors  # Required for coveralls reporting.
  - pip install -r requirements.txt
-  
-  - pip install pyflakes flake8 readme
+  
+  - pip install pyflakes flake8 readme coverage
+  
+
+  
+after_success:
+  
+  - coveralls
 # command to run tests, e.g. python setup.py test
 script: make lint && make test && if [[ $TRAVIS_PYTHON_VERSION == 2.6 ]]; then true; else make lint-readme; fi

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -1,3 +1,7 @@
+  
+# For testing
+  
+nose
+  
+coverage
+  
+
 # For dev
 sphinx

diff --git a/planemo/commands/cmd_tool_init.py b/planemo/commands/cmd_tool_init.py
@@ -152,6 +152,15 @@
    prompt=False,
    help="Add a Docker image identifier for this tool."
 )
+  
+@click.option(
+  
+    "--doi",
+  
+    type=click.STRING,
+  
+    default=None,
+  
+    multiple=True,
+  
+    prompt=False,
+  
+    help=("Supply a DOI (http://www.doi.org/) easing citation of the tool "
+  
+          "for Galxy users (e.g. 10.1101/014043).")
+  
+)
 @click.option(
    "--test_case",
    is_flag=True,

diff --git a/planemo/tool_builder.py b/planemo/tool_builder.py
@@ -62,6 +62,13 @@
        TODO: Fill in help.
 {%- endif %}
    ]]></help>
+  
+{%- if doi %}
+  
+    <citations>
+  
+{%- for single_doi in doi %}
+  
+        <citation type="doi">{{ single_doi }}</citation>
+  
+{%- endfor %}
+  
+    </citations>
+  
+{%- endif %}
 </tool>
 """

diff --git a/planemo_ext/galaxy/tools/lint.py b/planemo_ext/galaxy/tools/lint.py
@@ -17,10 +17,11 @@ def lint_xml(tool_xml, level=LEVEL_ALL, fail_level=LEVEL_WARN):
                lint_context.lint(module, name, value, tool_xml)
    found_warns = lint_context.found_warns
    found_errors = lint_context.found_errors
-  
-    if level == LEVEL_WARN and (found_warns or found_errors):
-  
-        return False
+  
+    if fail_level == LEVEL_WARN:
+  
+        lint_fail = (found_warns or found_errors)
    else:
-  
-        return found_errors
+  
+        lint_fail = found_errors
+  
+    return not lint_fail
 class LintContext(object):

diff --git a/tests/test_build_and_lint.py b/tests/test_build_and_lint.py
@@ -0,0 +1,62 @@
+  
+
+  
+from .test_utils import CliTestCase
+  
+
+  
+
+  
+INIT_COMMAND = [
+  
+    "tool_init", "--force",
+  
+    "--id", "seqtk_seq",
+  
+    "--name", "Convert to FASTA (seqtk)",
+  
+    "--requirement", "seqtk@1.0-r68",
+  
+    "--example_command", "seqtk seq -a 2.fastq > 2.fasta",
+  
+    "--example_input", "2.fastq",
+  
+    "--example_output", "2.fasta",
+  
+    "--test_case",
+  
+    "--help_text", "The help text.",
+  
+    "--doi", "10.1101/014043"
+  
+]
+  
+
+  
+
+  
+class BuildAndLintTestCase(CliTestCase):
+  
+
+  
+    def test_build_and_lint(self):
+  
+        with self._isolate():
+  
+            self._check_exit_code(_init_command())
+  
+            self._check_lint(exit_code=0)
+  
+
+  
+    def test_lint_fails_if_no_help(self):
+  
+        with self._isolate():
+  
+            self._check_exit_code(_init_command(help_text=False))
+  
+            self._check_lint(exit_code=1)
+  
+
+  
+    def test_lint_fails_if_no_test(self):
+  
+        with self._isolate():
+  
+            self._check_exit_code(_init_command(test_case=False))
+  
+            self._check_lint(exit_code=1)
+  
+
+  
+    def test_lint_fails_if_no_doi(self):
+  
+        with self._isolate():
+  
+            self._check_exit_code(_init_command(doi=False))
+  
+            self._check_lint(exit_code=1)
+  
+
+  
+    def _check_lint(self, exit_code=0):
+  
+        lint_cmd = ["lint", "--fail_level", "warn", "seqtk_seq.xml"]
+  
+        self._check_exit_code(lint_cmd, exit_code=exit_code)
+  
+
+  
+
+  
+def _init_command(test_case=True, help_text=True, doi=True):
+  
+    command = [
+  
+        "tool_init", "--force",
+  
+        "--id", "seqtk_seq",
+  
+        "--name", "Convert to FASTA (seqtk)",
+  
+        "--requirement", "seqtk@1.0-r68",
+  
+        "--example_command", "seqtk seq -a 2.fastq > 2.fasta",
+  
+        "--example_input", "2.fastq",
+  
+        "--example_output", "2.fasta"
+  
+    ]
+  
+    if test_case:
+  
+        command.append("--test_case")
+  
+    if help_text:
+  
+        command.extend(["--help_text", "The help text."])
+  
+    if doi:
+  
+        command.extend(["--doi", "10.1101/014043"])
+  
+    return command
diff --git a/tests/test_init_and_test.py b/tests/test_init_and_test.py
@@ -0,0 +1,11 @@
+  
+from .test_utils import CliTestCase
+  
+
+  
+
+  
+class InitAndTestTestCase(CliTestCase):
+  
+
+  
+    def test_init_and_test(self):
+  
+        with self._isolate():
+  
+            init_cmd = ["project_init", "--template", "demo", "basic"]
+  
+            self._check_exit_code(init_cmd)
+  
+            test_cmd = ["test", "--install_galaxy", "basic/cat.xml"]
+  
+            self._check_exit_code(test_cmd)
diff --git a/tests/test_planemo.py b/tests/test_planemo.py
@@ -8,48 +8,50 @@
 Tests for `planemo` module.
 """
-  
-from click.testing import CliRunner
+  
+from .test_utils import CliTestCase
+  
+from .test_utils import main
-  
-import unittest
-  
-from planemo import cli
-  
-
-  
-
-  
-# More information on testing click applications at following link.
-  
-# http://click.pocoo.org/3/testing/#basic-testing
-  
-class TestPlanemo(unittest.TestCase):
+  
+class TestPlanemo(CliTestCase):
    def test_commands_have_help(self):
-  
-        commands = cli.list_cmds()
-  
-        runner = CliRunner()
+  
+        commands = self._cli.list_cmds()
        for command in commands:
-  
-            planemo_cli = cli.planemo
-  
-            result = runner.invoke(planemo_cli, [command, "--help"])
-  
-            if result.exit_code != 0:
-  
-                message = "Planemo command %s has invalid --help." % command
-  
-                raise AssertionError(message)
+  
+            self._check_exit_code([command, "--help"])
    def test_responds_to_desired_commands(self):
-  
-        commands = cli.list_cmds()
+  
+        commands = self._cli.list_cmds()
        def assert_responds_to(command):
            assert command in commands, "No command %s" % command
-  
-        assert_responds_to("docker_shell")
-  
-        assert_responds_to("docker_build")
        assert_responds_to("brew_init")
        assert_responds_to("brew")
        assert_responds_to("brew_env")
        assert_responds_to("config_init")
+  
+        assert_responds_to("create_gist")
+  
+        assert_responds_to("docker_build")
+  
+        assert_responds_to("docker_shell")
        assert_responds_to("lint")
+  
+        assert_responds_to("normalize")
        assert_responds_to("project_init")
-  
-        assert_responds_to("shed_upload")
        assert_responds_to("serve")
+  
+        assert_responds_to("shed_diff")
+  
+        assert_responds_to("shed_download")
+  
+        assert_responds_to("shed_upload")
        assert_responds_to("syntax")
        assert_responds_to("test")
+  
+        assert_responds_to("tool_factory")
+  
+        assert_responds_to("tool_init")
        assert_responds_to("travis_before_install")
        assert_responds_to("travis_init")
+  
+    def test_planemo_version_command(self):
+  
+        self._check_exit_code(["--version"])
+  
+
+  
+    def test_planemo_help_command(self):
+  
+        self._check_exit_code(["--help"])
+  
+
 if __name__ == '__main__':
-  
-    unittest.main()
+  
+    main()
diff --git a/tests/test_utils.py b/tests/test_utils.py
@@ -0,0 +1,52 @@
+  
+""" Provide abstractions over click testing of the
+  
+app and unittest.
+  
+"""
+  
+from click.testing import CliRunner
+  
+
+  
+from unittest import (
+  
+    TestCase,
+  
+    main,
+  
+)
+  
+from planemo import cli
+  
+
+  
+EXIT_CODE_MESSAGE = ("Planemo command [%s] resulted in unexpected exit code "
+  
+                     "[%s], expected exit code [%s]]. Command output [%s]")
+  
+
+  
+
+  
+# More information on testing click applications at following link.
+  
+# http://click.pocoo.org/3/testing/#basic-testing
+  
+class CliTestCase(TestCase):
+  
+
+  
+    def setUp(self):  # noqa
+  
+        self._runner = CliRunner()
+  
+
+  
+    @property
+  
+    def _cli(self):
+  
+        return cli
+  
+
+  
+    def _isolate(self):
+  
+        return self._runner.isolated_filesystem()
+  
+
+  
+    def _invoke(self, command_list):
+  
+        planemo_cli = self._cli.planemo
+  
+        return self._runner.invoke(planemo_cli, command_list)
+  
+
+  
+    def _check_exit_code(self, command_list, exit_code=0):
+  
+        expected_exit_code = exit_code
+  
+        result = self._invoke(command_list)
+  
+        result_exit_code = result.exit_code
+  
+        if result_exit_code != expected_exit_code:
+  
+            message = EXIT_CODE_MESSAGE % (
+  
+                " ".join(command_list),
+  
+                result_exit_code,
+  
+                expected_exit_code,
+  
+                result.output,
+  
+            )
+  
+            raise AssertionError(message)
+  
+
+  
+
+  
+__all__ = [
+  
+    "TestCase",
+  
+    "CliTestCase",
+  
+    "main",
+  
+]