From 86a8c6a8690c1eda46031ab3f5f5f212645a3227 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 11 Mar 2018 22:32:45 -0400 Subject: [PATCH] Improve how various branches of Galaxy are testing. Annotate tests that include a significant aspect of Galaxy behavior testing and paramerize them to allow overridding the target branch via an environment variable. By default switch all of these to test master, but create new tox profiles and Travis targets that only run these tests and run for other branches (dev, release_18.01, and release_17.09 currently). Also drop the py27-quick set of tests from Travis - going to be future facing and be happy with the python 3 tests for the quick turn around / stable tests. --- .travis.yml | 4 +++- tests/test_cmd_serve.py | 8 +++++++- tests/test_cmd_test_conda.py | 10 ++++++++-- tests/test_galaxy_serve.py | 1 + tests/test_init_and_test.py | 20 ++++++++++---------- tests/test_run.py | 4 ++++ tests/test_utils.py | 22 ++++++++++++++++++++++ tox.ini | 14 +++++++++++--- 8 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0fd5aa8d4..d10dae8bd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,10 +6,12 @@ env: - TOX_ENV=py34-lint - TOX_ENV=py27-lint-readme - TOX_ENV=py27-lint-docs - - TOX_ENV=py27-quick - TOX_ENV=py34-quick - TOX_ENV=py27 - TOX_ENV=py34 + - TOX_ENV=py34-gx-1801 + - TOX_ENV=py34-gx-dev + - TOX_ENV=py34-gx-1709 - TOX_ENV=py27-lint-docstrings install: diff --git a/tests/test_cmd_serve.py b/tests/test_cmd_serve.py index f881a21ee..9b51dcaed 100644 --- a/tests/test_cmd_serve.py +++ b/tests/test_cmd_serve.py @@ -9,9 +9,11 @@ cli_daemon_service, CliTestCase, launch_and_wait_for_service, + mark, PROJECT_TEMPLATES_DIR, skip_if_environ, skip_unless_environ, + target_galaxy_branch, TEST_DATA_DIR, TEST_REPOS_DIR, ) @@ -22,10 +24,12 @@ class ServeTestCase(CliTestCase): @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_serve(self): self._launch_thread_and_wait(self._run) @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_serve_daemon(self): extra_args = ["--daemon", "--pid_file", self._pid_file] self._launch_thread_and_wait(self._run, extra_args) @@ -35,6 +39,7 @@ def test_serve_daemon(self): kill_pid_file(self._pid_file) @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_serve_workflow(self): random_lines = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml") cat = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "cat.xml") @@ -54,6 +59,7 @@ def test_serve_workflow(self): assert len(user_gi.workflows.get_workflows()) == 1 @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_shed_serve(self): extra_args = ["--daemon", "--pid_file", self._pid_file, "--shed_target", "toolshed"] fastqc_path = os.path.join(TEST_REPOS_DIR, "fastqc") @@ -124,7 +130,7 @@ def _run(self, serve_args=[], serve_cmd="serve"): def _serve_command_list(self, serve_args=[], serve_cmd="serve"): test_cmd = [ serve_cmd, - "--install_galaxy", + "--galaxy_branch", target_galaxy_branch(), "--no_dependency_resolution", "--port", str(self._port), diff --git a/tests/test_cmd_test_conda.py b/tests/test_cmd_test_conda.py index cb163dca0..1d6991440 100644 --- a/tests/test_cmd_test_conda.py +++ b/tests/test_cmd_test_conda.py @@ -3,8 +3,10 @@ from .test_utils import ( CliTestCase, + mark, PROJECT_TEMPLATES_DIR, skip_if_environ, + target_galaxy_branch, TEST_REPOS_DIR, TEST_TOOLS_DIR, ) @@ -14,13 +16,14 @@ class CmdTestCondaTestCase(CliTestCase): """Integration tests for the ``test`` command.""" @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_conda_dependencies_by_default(self): with self._isolate(): bwa_test = os.path.join(PROJECT_TEMPLATES_DIR, "conda_testing", "bwa.xml") test_command = [ "--verbose", "test", - "--galaxy_branch", "dev", + "--galaxy_branch", target_galaxy_branch(), bwa_test, ] self._check_exit_code(test_command, exit_code=0) @@ -41,6 +44,7 @@ def test_conda_dependencies_explicit_resolution(self): self._check_exit_code(test_command, exit_code=0) @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_conda_dependencies_version(self): """Test tool with wrong version and ensure it fails.""" with self._isolate(): @@ -49,6 +53,7 @@ def test_conda_dependencies_version(self): test_command = [ "--verbose", "test", + "--galaxy_branch", target_galaxy_branch(), "--conda_dependency_resolution", "--conda_auto_install", "--conda_auto_init", @@ -57,6 +62,7 @@ def test_conda_dependencies_version(self): self._check_exit_code(test_command, exit_code=1) @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_local_conda_dependencies_version(self): """Test a tool that requires local package builds.""" with self._isolate(): @@ -75,7 +81,7 @@ def test_local_conda_dependencies_version(self): self._check_exit_code(conda_install_command) test_command = [ "test", - "--galaxy_branch", "release_17.09", + "--galaxy_branch", target_galaxy_branch(), fleeqtk_tool, ] self._check_exit_code(test_command) diff --git a/tests/test_galaxy_serve.py b/tests/test_galaxy_serve.py index 345d12fea..7638c474d 100644 --- a/tests/test_galaxy_serve.py +++ b/tests/test_galaxy_serve.py @@ -50,6 +50,7 @@ def test_serve_daemon(self): timeout=.1, ) + @skip_if_environ("PLANEMO_SKIP_REDUNDANT_TESTS") # redundant with test_cmd_serve -> test_serve_workflow @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") def test_serve_workflow(self): """Test serving a galaxy workflow via a daemon Galaxy process.""" diff --git a/tests/test_init_and_test.py b/tests/test_init_and_test.py index 7c3c7c2ac..b1691adb4 100644 --- a/tests/test_init_and_test.py +++ b/tests/test_init_and_test.py @@ -1,23 +1,23 @@ from .test_utils import ( CliTestCase, + mark, skip_if_environ, + target_galaxy_branch, ) class InitAndTestTestCase(CliTestCase): @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") - def test_init_and_test_master(self): - self.__run_commands() - - @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") - def test_init_and_test_dev(self): - self.__run_commands(test_args=["--galaxy_branch", "dev"]) - - def __run_commands(self, test_args=[]): + @mark.tests_galaxy_branch + def test_init_and_test(self): with self._isolate(): init_cmd = ["project_init", "--template", "demo", "basic"] self._check_exit_code(init_cmd) - test_cmd = ["test", "--install_galaxy"] + test_args - test_cmd += ["basic/cat.xml"] + test_cmd = [ + "test", + "--no_dependency_resolution", + "--galaxy_branch", target_galaxy_branch(), + "basic/cat.xml" + ] self._check_exit_code(test_cmd) diff --git a/tests/test_run.py b/tests/test_run.py index ae960f6b6..fc380deb7 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -4,8 +4,10 @@ from .test_utils import ( CliTestCase, CWL_DRAFT3_DIR, + mark, PROJECT_TEMPLATES_DIR, skip_if_environ, + target_galaxy_branch, TEST_DATA_DIR, ) @@ -48,6 +50,7 @@ def test_run_cat_cwltool_more_options(self): self._check_exit_code(test_cmd) @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS") + @mark.tests_galaxy_branch def test_run_gxtool_randomlines(self): with self._isolate(): tool_path = os.path.join(PROJECT_TEMPLATES_DIR, "demo", "randomlines.xml") @@ -56,6 +59,7 @@ def test_run_gxtool_randomlines(self): "--verbose", "run", "--no_dependency_resolution", + "--galaxy_branch", target_galaxy_branch(), tool_path, job_path, ] diff --git a/tests/test_utils.py b/tests/test_utils.py index 3a7568d58..e64f4bb29 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -24,6 +24,12 @@ setup_mock_shed, ) +try: + import pytest +except ImportError: + pytest = None + from nose.plugins.attrib import attr + if version_info < (2, 7): from unittest2 import TestCase, skip PRE_PYTHON_27 = True @@ -47,6 +53,18 @@ NON_ZERO_EXIT_CODE = object() +class MarkGenerator(object): + + def __getattr__(self, name): + if pytest: + return getattr(pytest.mark, name) + else: + return attr(name) + + +mark = MarkGenerator() + + # More information on testing click applications at following link. # http://click.pocoo.org/3/testing/#basic-testing class CliTestCase(TestCase): @@ -203,6 +221,10 @@ def skip_unless_python_2_7(): return skip("Python 2.7 required for test.") +def target_galaxy_branch(): + return os.environ.get("PLANEMO_TEST_GALAXY_BRANCH", "master") + + def test_context(): context = cli.Context() context.planemo_directory = "/tmp/planemo-test-workspace" diff --git a/tox.ini b/tox.ini index ed615b239..ca2c5ab2b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,14 @@ # TODO: implement doc linting [tox] -envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35} +envlist = py{27,34}-lint, py{27,34,35}-quick, py27-lint-imports, py27-lint-docstrings, py27-lint-readme, py27-lint-docs, py{27,34,35}, py{27,34,35}-gx-{master,dev,1801,1709,1705} source_dir = planemo test_dir = tests [testenv] -commands = {envpython} setup.py nosetests [] -passenv = PLANEMO_* +commands = nosetests +passenv = + PLANEMO_* + NOSE_* deps = -rrequirements.txt nose @@ -15,6 +17,12 @@ deps = setenv = quick: PLANEMO_SKIP_SLOW_TESTS=1 quick: PLANEMO_SKIP_GALAXY_TESTS=1 + gx: NOSE_ATTR=tests_galaxy_branch + master: PLANEMO_TEST_GALAXY_BRANCH=master + dev: PLANEMO_TEST_GALAXY_BRANCH=dev + 1801: PLANEMO_TEST_GALAXY_BRANCH=release_18.01 + 1709: PLANEMO_TEST_GALAXY_BRANCH=release_17.09 + 1705: PLANEMO_TEST_GALAXY_BRANCH=release_17.05 [testenv:py27-lint] commands = flake8 {[tox]source_dir} {[tox]test_dir}