Merge pull request #800 from jmchilton/gxtesting

Improve how various branches of Galaxy are testing.

.travis.yml

@@ -6,10 +6,12 @@ env:
   - TOX_ENV=py34-lint
   - TOX_ENV=py27-lint-readme
   - TOX_ENV=py27-lint-docs
-  - TOX_ENV=py27-quick
   - TOX_ENV=py34-quick
   - TOX_ENV=py27
   - TOX_ENV=py34
+  - TOX_ENV=py34-gx-1801
+  - TOX_ENV=py34-gx-dev
+  - TOX_ENV=py34-gx-1709
   - TOX_ENV=py27-lint-docstrings
 
 install:

tests/test_cmd_serve.py

@@ -9,9 +9,11 @@
     cli_daemon_service,
     CliTestCase,
     launch_and_wait_for_service,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
     skip_unless_environ,
+    target_galaxy_branch,
     TEST_DATA_DIR,
     TEST_REPOS_DIR,
 )
@@ -22,10 +24,12 @@
 class ServeTestCase(CliTestCase):
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve(self):
         self._launch_thread_and_wait(self._run)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve_daemon(self):
         extra_args = ["--daemon", "--pid_file", self._pid_file]
         self._launch_thread_and_wait(self._run, extra_args)
@@ -35,6 +39,7 @@ def test_serve_daemon(self):
         kill_pid_file(self._pid_file)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve_workflow(self):
         random_lines = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml")
         cat = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "cat.xml")
@@ -54,6 +59,7 @@ def test_serve_workflow(self):
         assert len(user_gi.workflows.get_workflows()) == 1
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_shed_serve(self):
         extra_args = ["--daemon", "--pid_file", self._pid_file, "--shed_target", "toolshed"]
         fastqc_path = os.path.join(TEST_REPOS_DIR, "fastqc")
@@ -124,7 +130,7 @@ def _run(self, serve_args=[], serve_cmd="serve"):
     def _serve_command_list(self, serve_args=[], serve_cmd="serve"):
         test_cmd = [
             serve_cmd,
-            "--install_galaxy",
+            "--galaxy_branch", target_galaxy_branch(),
             "--no_dependency_resolution",
             "--port",
             str(self._port),

tests/test_cmd_test_conda.py

@@ -3,8 +3,10 @@
 
 from .test_utils import (
     CliTestCase,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
+    target_galaxy_branch,
     TEST_REPOS_DIR,
     TEST_TOOLS_DIR,
 )
@@ -14,13 +16,14 @@ class CmdTestCondaTestCase(CliTestCase):
     """Integration tests for the ``test`` command."""
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_conda_dependencies_by_default(self):
         with self._isolate():
             bwa_test = os.path.join(PROJECT_TEMPLATES_DIR, "conda_testing", "bwa.xml")
             test_command = [
                 "--verbose",
                 "test",
-                "--galaxy_branch", "dev",
+                "--galaxy_branch", target_galaxy_branch(),
                 bwa_test,
             ]
             self._check_exit_code(test_command, exit_code=0)
@@ -41,6 +44,7 @@ def test_conda_dependencies_explicit_resolution(self):
             self._check_exit_code(test_command, exit_code=0)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_conda_dependencies_version(self):
         """Test tool with wrong version and ensure it fails."""
         with self._isolate():
@@ -49,6 +53,7 @@ def test_conda_dependencies_version(self):
             test_command = [
                 "--verbose",
                 "test",
+                "--galaxy_branch", target_galaxy_branch(),
                 "--conda_dependency_resolution",
                 "--conda_auto_install",
                 "--conda_auto_init",
@@ -57,6 +62,7 @@ def test_conda_dependencies_version(self):
             self._check_exit_code(test_command, exit_code=1)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_local_conda_dependencies_version(self):
         """Test a tool that requires local package builds."""
         with self._isolate():
@@ -75,7 +81,7 @@ def test_local_conda_dependencies_version(self):
             self._check_exit_code(conda_install_command)
             test_command = [
                 "test",
-                "--galaxy_branch", "release_17.09",
+                "--galaxy_branch", target_galaxy_branch(),
                 fleeqtk_tool,
             ]
             self._check_exit_code(test_command)

tests/test_galaxy_serve.py

@@ -50,6 +50,7 @@ def test_serve_daemon(self):
             timeout=.1,
         )
 
+    @skip_if_environ("PLANEMO_SKIP_REDUNDANT_TESTS")  # redundant with test_cmd_serve -> test_serve_workflow
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
     def test_serve_workflow(self):
         """Test serving a galaxy workflow via a daemon Galaxy process."""

tests/test_init_and_test.py

@@ -1,23 +1,23 @@
 from .test_utils import (
     CliTestCase,
+    mark,
     skip_if_environ,
+    target_galaxy_branch,
 )
 
 
 class InitAndTestTestCase(CliTestCase):
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
-    def test_init_and_test_master(self):
-        self.__run_commands()
-
-    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
-    def test_init_and_test_dev(self):
-        self.__run_commands(test_args=["--galaxy_branch", "dev"])
-
-    def __run_commands(self, test_args=[]):
+    @mark.tests_galaxy_branch
+    def test_init_and_test(self):
         with self._isolate():
             init_cmd = ["project_init", "--template", "demo", "basic"]
             self._check_exit_code(init_cmd)
-            test_cmd = ["test", "--install_galaxy"] + test_args
-            test_cmd += ["basic/cat.xml"]
+            test_cmd = [
+                "test",
+                "--no_dependency_resolution",
+                "--galaxy_branch", target_galaxy_branch(),
+                "basic/cat.xml"
+            ]
             self._check_exit_code(test_cmd)

tests/test_run.py

@@ -4,8 +4,10 @@
 from .test_utils import (
     CliTestCase,
     CWL_DRAFT3_DIR,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
+    target_galaxy_branch,
     TEST_DATA_DIR,
 )
 
@@ -48,6 +50,7 @@ def test_run_cat_cwltool_more_options(self):
             self._check_exit_code(test_cmd)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_run_gxtool_randomlines(self):
         with self._isolate():
             tool_path = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml")
@@ -56,6 +59,7 @@ def test_run_gxtool_randomlines(self):
                 "--verbose",
                 "run",
                 "--no_dependency_resolution",
+                "--galaxy_branch", target_galaxy_branch(),
                 tool_path,
                 job_path,
             ]

tests/test_utils.py

@@ -24,6 +24,12 @@
     setup_mock_shed,
 )
 
+try:
+    import pytest
+except ImportError:
+    pytest = None
+    from nose.plugins.attrib import attr
+
 if version_info < (2, 7):
     from unittest2 import TestCase, skip
     PRE_PYTHON_27 = True
@@ -47,6 +53,18 @@
 NON_ZERO_EXIT_CODE = object()
 
 
+class MarkGenerator(object):
+
+    def __getattr__(self, name):
+        if pytest:
+            return getattr(pytest.mark, name)
+        else:
+            return attr(name)
+
+
+mark = MarkGenerator()
+
+
 # More information on testing click applications at following link.
 # http://click.pocoo.org/3/testing/#basic-testing
 class CliTestCase(TestCase):
@@ -203,6 +221,10 @@ def skip_unless_python_2_7():
     return skip("Python 2.7 required for test.")
 
 
+def target_galaxy_branch():
+    return os.environ.get("PLANEMO_TEST_GALAXY_BRANCH", "master")
+
+
 def test_context():
     context = cli.Context()
     context.planemo_directory = "/tmp/planemo-test-workspace"

tox.ini

@@ -1,12 +1,14 @@
 # TODO: implement doc linting
 [tox]
-envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35}
+envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35}, py{27,34,35}-gx-{master,dev,1801,1709,1705}
 source_dir = planemo
 test_dir = tests
 
 [testenv]
-commands = {envpython} setup.py nosetests []
-passenv = PLANEMO_*
+commands = nosetests
+passenv = 
+    PLANEMO_*
+    NOSE_*
 deps =
     -rrequirements.txt
     nose
@@ -15,6 +17,12 @@ deps =
 setenv =
     quick: PLANEMO_SKIP_SLOW_TESTS=1
     quick: PLANEMO_SKIP_GALAXY_TESTS=1
+    gx: NOSE_ATTR=tests_galaxy_branch
+    master: PLANEMO_TEST_GALAXY_BRANCH=master
+    dev: PLANEMO_TEST_GALAXY_BRANCH=dev
+    1801: PLANEMO_TEST_GALAXY_BRANCH=release_18.01
+    1709: PLANEMO_TEST_GALAXY_BRANCH=release_17.09
+    1705: PLANEMO_TEST_GALAXY_BRANCH=release_17.05
 
 [testenv:py27-lint]
 commands = flake8 {[tox]source_dir} {[tox]test_dir}