Improve how various branches of Galaxy are testing.

Annotate tests that include a significant aspect of Galaxy behavior testing and paramerize them to allow overridding the target branch via an environment variable. By default switch all of these to test master, but create new tox profiles and Travis targets that only run these tests and run for other branches (dev, release_18.01, and release_17.09 currently).

Also drop the py27-quick set of tests from Travis - going to be future facing and be happy with the python 3 tests for the quick turn around / stable tests.

.travis.yml

@@ -6,10 +6,12 @@ env:
   - TOX_ENV=py34-lint
   - TOX_ENV=py27-lint-readme
   - TOX_ENV=py27-lint-docs
-  - TOX_ENV=py27-quick
   - TOX_ENV=py34-quick
   - TOX_ENV=py27
   - TOX_ENV=py34
+  - TOX_ENV=py34-gx-1801
+  - TOX_ENV=py34-gx-dev
+  - TOX_ENV=py34-gx-1709
   - TOX_ENV=py27-lint-docstrings
 
 install:

tests/test_cmd_serve.py

@@ -9,9 +9,11 @@
     cli_daemon_service,
     CliTestCase,
     launch_and_wait_for_service,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
     skip_unless_environ,
+    target_galaxy_branch,
     TEST_DATA_DIR,
     TEST_REPOS_DIR,
 )
@@ -22,10 +24,12 @@
 class ServeTestCase(CliTestCase):
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve(self):
         self._launch_thread_and_wait(self._run)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve_daemon(self):
         extra_args = ["--daemon", "--pid_file", self._pid_file]
         self._launch_thread_and_wait(self._run, extra_args)
@@ -35,6 +39,7 @@ def test_serve_daemon(self):
         kill_pid_file(self._pid_file)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_serve_workflow(self):
         random_lines = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml")
         cat = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "cat.xml")
@@ -54,6 +59,7 @@ def test_serve_workflow(self):
         assert len(user_gi.workflows.get_workflows()) == 1
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_shed_serve(self):
         extra_args = ["--daemon", "--pid_file", self._pid_file, "--shed_target", "toolshed"]
         fastqc_path = os.path.join(TEST_REPOS_DIR, "fastqc")
@@ -124,7 +130,7 @@ def _run(self, serve_args=[], serve_cmd="serve"):
     def _serve_command_list(self, serve_args=[], serve_cmd="serve"):
         test_cmd = [
             serve_cmd,
-            "--install_galaxy",
+            "--galaxy_branch", target_galaxy_branch(),
             "--no_dependency_resolution",
             "--port",
             str(self._port),

tests/test_cmd_test_conda.py

@@ -3,8 +3,10 @@
 
 from .test_utils import (
     CliTestCase,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
+    target_galaxy_branch,
     TEST_REPOS_DIR,
     TEST_TOOLS_DIR,
 )
@@ -14,13 +16,14 @@ class CmdTestCondaTestCase(CliTestCase):
     """Integration tests for the ``test`` command."""
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_conda_dependencies_by_default(self):
         with self._isolate():
             bwa_test = os.path.join(PROJECT_TEMPLATES_DIR, "conda_testing", "bwa.xml")
             test_command = [
                 "--verbose",
                 "test",
-                "--galaxy_branch", "dev",
+                "--galaxy_branch", target_galaxy_branch(),
                 bwa_test,
             ]
             self._check_exit_code(test_command, exit_code=0)
@@ -41,6 +44,7 @@ def test_conda_dependencies_explicit_resolution(self):
             self._check_exit_code(test_command, exit_code=0)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_conda_dependencies_version(self):
         """Test tool with wrong version and ensure it fails."""
         with self._isolate():
@@ -49,6 +53,7 @@ def test_conda_dependencies_version(self):
             test_command = [
                 "--verbose",
                 "test",
+                "--galaxy_branch", target_galaxy_branch(),
                 "--conda_dependency_resolution",
                 "--conda_auto_install",
                 "--conda_auto_init",
@@ -57,6 +62,7 @@ def test_conda_dependencies_version(self):
             self._check_exit_code(test_command, exit_code=1)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_local_conda_dependencies_version(self):
         """Test a tool that requires local package builds."""
         with self._isolate():
@@ -75,7 +81,7 @@ def test_local_conda_dependencies_version(self):
             self._check_exit_code(conda_install_command)
             test_command = [
                 "test",
-                "--galaxy_branch", "release_17.09",
+                "--galaxy_branch", target_galaxy_branch(),
                 fleeqtk_tool,
             ]
             self._check_exit_code(test_command)

tests/test_galaxy_serve.py

@@ -50,6 +50,7 @@ def test_serve_daemon(self):
             timeout=.1,
         )
 
+    @skip_if_environ("PLANEMO_SKIP_REDUNDANT_TESTS")  # redundant with test_cmd_serve -> test_serve_workflow
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
     def test_serve_workflow(self):
         """Test serving a galaxy workflow via a daemon Galaxy process."""

tests/test_init_and_test.py

@@ -1,23 +1,23 @@
 from .test_utils import (
     CliTestCase,
+    mark,
     skip_if_environ,
+    target_galaxy_branch,
 )
 
 
 class InitAndTestTestCase(CliTestCase):
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
-    def test_init_and_test_master(self):
-        self.__run_commands()
-
-    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
-    def test_init_and_test_dev(self):
-        self.__run_commands(test_args=["--galaxy_branch", "dev"])
-
-    def __run_commands(self, test_args=[]):
+    @mark.tests_galaxy_branch
+    def test_init_and_test(self):
         with self._isolate():
             init_cmd = ["project_init", "--template", "demo", "basic"]
             self._check_exit_code(init_cmd)
-            test_cmd = ["test", "--install_galaxy"] + test_args
-            test_cmd += ["basic/cat.xml"]
+            test_cmd = [
+                "test",
+                "--no_dependency_resolution",
+                "--galaxy_branch", target_galaxy_branch(),
+                "basic/cat.xml"
+            ]
             self._check_exit_code(test_cmd)

tests/test_run.py

@@ -4,8 +4,10 @@
 from .test_utils import (
     CliTestCase,
     CWL_DRAFT3_DIR,
+    mark,
     PROJECT_TEMPLATES_DIR,
     skip_if_environ,
+    target_galaxy_branch,
     TEST_DATA_DIR,
 )
 
@@ -48,6 +50,7 @@ def test_run_cat_cwltool_more_options(self):
             self._check_exit_code(test_cmd)
 
     @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    @mark.tests_galaxy_branch
     def test_run_gxtool_randomlines(self):
         with self._isolate():
             tool_path = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml")
@@ -56,6 +59,7 @@ def test_run_gxtool_randomlines(self):
                 "--verbose",
                 "run",
                 "--no_dependency_resolution",
+                "--galaxy_branch", target_galaxy_branch(),
                 tool_path,
                 job_path,
             ]

tests/test_utils.py

@@ -24,6 +24,12 @@
     setup_mock_shed,
 )
 
+try:
+    import pytest
+except ImportError:
+    pytest = None
+    from nose.plugins.attrib import attr
+
 if version_info < (2, 7):
     from unittest2 import TestCase, skip
     PRE_PYTHON_27 = True
@@ -47,6 +53,18 @@
 NON_ZERO_EXIT_CODE = object()
 
 
+class MarkGenerator(object):
+
+    def __getattr__(self, name):
+        if pytest:
+            return getattr(pytest.mark, name)
+        else:
+            return attr(name)
+
+
+mark = MarkGenerator()
+
+
 # More information on testing click applications at following link.
 # http://click.pocoo.org/3/testing/#basic-testing
 class CliTestCase(TestCase):
@@ -203,6 +221,10 @@ def skip_unless_python_2_7():
     return skip("Python 2.7 required for test.")
 
 
+def target_galaxy_branch():
+    return os.environ.get("PLANEMO_TEST_GALAXY_BRANCH", "master")
+
+
 def test_context():
     context = cli.Context()
     context.planemo_directory = "/tmp/planemo-test-workspace"

tox.ini

@@ -1,12 +1,14 @@
 # TODO: implement doc linting
 [tox]
-envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35}
+envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35}, py{27,34,35}-gx-{master,dev,1801,1709,1705}
 source_dir = planemo
 test_dir = tests
 
 [testenv]
-commands = {envpython} setup.py nosetests []
-passenv = PLANEMO_*
+commands = nosetests
+passenv = 
+    PLANEMO_*
+    NOSE_*
 deps =
     -rrequirements.txt
     nose
@@ -15,6 +17,12 @@ deps =
 setenv =
     quick: PLANEMO_SKIP_SLOW_TESTS=1
     quick: PLANEMO_SKIP_GALAXY_TESTS=1
+    gx: NOSE_ATTR=tests_galaxy_branch
+    master: PLANEMO_TEST_GALAXY_BRANCH=master
+    dev: PLANEMO_TEST_GALAXY_BRANCH=dev
+    1801: PLANEMO_TEST_GALAXY_BRANCH=release_18.01
+    1709: PLANEMO_TEST_GALAXY_BRANCH=release_17.09
+    1705: PLANEMO_TEST_GALAXY_BRANCH=release_17.05
 
 [testenv:py27-lint]
 commands = flake8 {[tox]source_dir} {[tox]test_dir}