Update planemo for Galaxy wheels.

That got out of hand quickly.
galaxyproject · Nov 11, 2015 · 25ef0d569e94c4d9f03e6f1225805eef6b65d9b5 · 25ef0d5
1 parent cbc2806
commit 25ef0d569e94c4d9f03e6f1225805eef6b65d9b5
Unified Split

Showing with 115 additions and 22 deletions.

+16 −11 planemo/galaxy_config.py

+74 −2 planemo/galaxy_run.py

+11 −3 planemo/galaxy_serve.py

+9 −5 planemo/galaxy_test/actions.py

+1 −1 planemo/galaxy_test/structures.py

+4 −0 planemo/io.py
diff --git a/planemo/galaxy_config.py b/planemo/galaxy_config.py
@@ -16,6 +16,7 @@
 from planemo import galaxy_run
 from planemo.io import warn
 from planemo.io import shell
+  
+from planemo.io import shell_join
 from planemo.io import write_file
 from planemo.io import kill_pid_file
 from planemo.io import wait_on
@@ -92,7 +93,9 @@
 DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
 LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
-  
-PIP_INSTALL_CMD = "[ -d .venv ] && . .venv/bin/activate && pip install %s"
+  
+PIP_INSTALL_CMD = "pip install %s"
+  
+
+  
+COMMAND_STARTUP_COMMAND = "./scripts/common_startup.sh ${COMMON_STARTUP_ARGS}"
 FAILED_TO_FIND_GALAXY_EXCEPTION = (
    "Failed to find Galaxy root directory - please explicitly specify one "
@@ -523,21 +526,21 @@ def _install_galaxy(ctx, config_directory, kwds):
    if not kwds.get("no_cache_galaxy", False):
        _install_galaxy_via_git(ctx, config_directory, kwds)
    else:
-  
-        _install_galaxy_via_download(config_directory, kwds)
+  
+        _install_galaxy_via_download(ctx, config_directory, kwds)
-  
-def _install_galaxy_via_download(config_directory, kwds):
+  
+def _install_galaxy_via_download(ctx, config_directory, kwds):
    branch = _galaxy_branch(kwds)
    tar_cmd = "tar -zxvf %s" % branch
    command = galaxy_run.DOWNLOAD_GALAXY + "; %s | tail" % tar_cmd
-  
-    _install_with_command(config_directory, command, kwds)
+  
+    _install_with_command(ctx, config_directory, command, kwds)
 def _install_galaxy_via_git(ctx, config_directory, kwds):
    gx_repo = _ensure_galaxy_repository_available(ctx, kwds)
    branch = _galaxy_branch(kwds)
    command = git.command_clone(ctx, gx_repo, "galaxy-dev", branch=branch)
-  
-    _install_with_command(config_directory, command, kwds)
+  
+    _install_with_command(ctx, config_directory, command, kwds)
 def _build_eggs_cache(ctx, env, kwds):
@@ -557,7 +560,7 @@ def _galaxy_branch(kwds):
    return branch
-  
-def _install_with_command(config_directory, command, kwds):
+  
+def _install_with_command(ctx, config_directory, command, kwds):
    # TODO: --watchdog
    pip_installs = []
    if kwds.get("cwl", False):
@@ -566,15 +569,17 @@ def _install_with_command(config_directory, command, kwds):
        pip_install_command = PIP_INSTALL_CMD % " ".join(pip_installs)
    else:
        pip_install_command = ""
-  
-    install_cmds = [
+  
+    setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
+  
+    install_cmd = shell_join(
        "cd %s" % config_directory,
        command,
        "cd galaxy-dev",
-  
-        "type virtualenv >/dev/null 2>&1 && virtualenv .venv",
+  
+        setup_venv_command,
        pip_install_command,
-  
-        galaxy_run.ACTIVATE_COMMAND,
-  
-    ]
-  
-    shell(";".join([c for c in install_cmds if c]))
+  
+        galaxy_run.setup_common_startup_args(),
+  
+        COMMAND_STARTUP_COMMAND,
+  
+    )
+  
+    shell(install_cmd)
 def _ensure_galaxy_repository_available(ctx, kwds):

diff --git a/planemo/galaxy_run.py b/planemo/galaxy_run.py
@@ -1,16 +1,88 @@
-  
-from planemo.io import info
+  
+import os
+  
+import string
+  
+
+  
+from planemo.io import info, shell_join
 from galaxy.tools.deps.commands import shell
 # Activate galaxy's virtualenv if present (needed for tests say but not for
 # server because run.sh does this).
-  
-ACTIVATE_COMMAND = "[ -e .venv ] && . .venv/bin/activate"
+  
+ACTIVATE_COMMAND = "[ -e $GALAXY_VIRTUAL_ENV ] && . $GALAXY_VIRTUAL_ENV/bin/activate"
+  
+CREATE_COMMAND = shell_join(
+  
+    'if [ ! -e $GALAXY_VIRTUAL_ENV ]',
+  
+    ' then type virtualenv >/dev/null 2>&1 && virtualenv $GALAXY_VIRTUAL_ENV',
+  
+    ' else echo "Reusing existing virtualenv $GALAXY_VIRTUAL_ENV"',
+  
+    ' fi',
+  
+)
+  
+PRINT_VENV_COMMAND = shell_join(
+  
+    'echo "Set \$GALAXY_VIRTUAL_ENV to $GALAXY_VIRTUAL_ENV"',
+  
+    'if [ -e $GALAXY_VIRTUAL_ENV ]',
+  
+    'then echo "Virtual environment directory exists."',
+  
+    'else echo "Virtual environment directory does not exist."',
+  
+    'fi',
+  
+)
+  
+
 # TODO: Mac-y curl variant of this.
 DOWNLOAD_GALAXY = (
    "wget https://codeload.github.com/galaxyproject/galaxy/tar.gz/dev"
 )
+  
+CACHED_VIRTUAL_ENV_COMMAND = ("if [ -d .venv ] || [ -f dist-eggs.ini ];"
+  
+                              " then GALAXY_VIRTUAL_ENV=.venv; "
+  
+                              " else GALAXY_VIRTUAL_ENV=%s; fi")
+  
+UNCACHED_VIRTUAL_ENV_COMMAND = "GALAXY_VIRTUAL_ENV=.venv"
+  
+
+  
+
+  
+def setup_venv(ctx, kwds):
+  
+    return shell_join(
+  
+        locate_galaxy_virtualenv(ctx, kwds),
+  
+        PRINT_VENV_COMMAND if ctx.verbose else None,
+  
+        CREATE_COMMAND,
+  
+        PRINT_VENV_COMMAND if ctx.verbose else None,
+  
+        ACTIVATE_COMMAND,
+  
+    )
+  
+
+  
+
+  
+def locate_galaxy_virtualenv(ctx, kwds):
+  
+    if not kwds.get("no_cache_galaxy", False):
+  
+        workspace = ctx.workspace
+  
+        shared_venv_path = os.path.join(workspace, "gx_venv")
+  
+        venv_command = CACHED_VIRTUAL_ENV_COMMAND % shared_venv_path
+  
+    else:
+  
+        venv_command = UNCACHED_VIRTUAL_ENV_COMMAND
+  
+    return shell_join(
+  
+        venv_command,
+  
+        "export GALAXY_VIRTUAL_ENV",
+  
+    )
+  
+
+  
+
+  
+def shell_if_wheels(command):
+  
+    """ Take a shell command and convert it to shell command that runs
+  
+    only if Galaxy is new enough to use wheels.
+  
+    """
+  
+    return "$(grep -q 'skip-venv' run_tests.sh) && %s" % command
+  
+
+  
+
+  
+def setup_common_startup_args():
+  
+    return set_variable_if_wheels(
+  
+        "COMMON_STARTUP_ARGS", "--skip-venv --dev-wheels"
+  
+    )
+  
+
+  
+
+  
+def set_variable_if_wheels(var, if_wheels_val, else_val=""):
+  
+    var_command = '${var}=${else_val}; '
+  
+    var_command += shell_if_wheels(
+  
+        '${var}="${if_wheels_val}"; '
+  
+    )
+  
+    var_command += "export ${var}"
+  
+    var_command += '; echo "Set ${var} to ${${var}}"'
+  
+    return string.Template(var_command).safe_substitute(
+  
+        var=var,
+  
+        if_wheels_val=if_wheels_val,
+  
+        else_val=else_val,
+  
+    )
+  
+
 def run_galaxy_command(ctx, command, env, action, daemon=False):
    message = "%s with command [%s]" % (action, command)

diff --git a/planemo/galaxy_serve.py b/planemo/galaxy_serve.py
@@ -17,18 +17,26 @@ def serve(ctx, paths, **kwds):
    with galaxy_config.galaxy_config(ctx, paths, **kwds) as config:
        # TODO: Allow running dockerized Galaxy here instead.
+  
+        setup_common_startup_args = galaxy_run.set_variable_if_wheels(
+  
+            "COMMON_STARTUP_ARGS", "--skip-venv"
+  
+        )
+  
+        setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
        run_script = os.path.join(config.galaxy_root, "run.sh")
+  
+        run_script += " $COMMON_STARTUP_ARGS"
        if daemon:
            run_script += " --daemon --wait"
            config.env["GALAXY_RUN_ALL"] = "1"
        else:
            run_script += " --server-name '%s' --reload" % config.server_name
        server_ini = os.path.join(config.config_directory, "galaxy.ini")
        config.env["GALAXY_CONFIG_FILE"] = server_ini
-  
-        cmds = [
+  
+        cd_to_galaxy_command = "cd %s" % config.galaxy_root
+  
+        cmd = io.shell_join(
+  
+            cd_to_galaxy_command,
+  
+            setup_common_startup_args,
+  
+            setup_venv_command,
            run_script,
-  
-        ]
-  
-        cmd = "; ".join(cmds)
+  
+        )
        action = "Starting galaxy"
        galaxy_run.run_galaxy_command(
            ctx,

diff --git a/planemo/galaxy_test/actions.py b/planemo/galaxy_test/actions.py
@@ -3,7 +3,7 @@
 import click
 from . import structures as test_structures
-  
-from planemo.io import info, warn
+  
+from planemo.io import info, warn, shell_join
 from planemo import galaxy_run
 from planemo.reports import build_report
@@ -51,12 +51,16 @@ def run_in_config(ctx, config, **kwds):
        failed=kwds.get("failed", False),
        installed=kwds.get("installed", False),
    ).build()
-  
-    cmd = "; ".join([
+  
+    setup_common_startup_args = galaxy_run.set_variable_if_wheels(
+  
+        "COMMON_STARTUP_ARGS", "--skip-venv --skip-common-startup"
+  
+    )
+  
+    setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
+  
+    cmd = shell_join(
        cd_to_galaxy_command,
-  
-        galaxy_run.ACTIVATE_COMMAND,  # TODO: this should be moved to
-  
-                                      # run_tests.sh to match run.sh.
+  
+        setup_common_startup_args,
+  
+        setup_venv_command,
        test_cmd,
-  
-    ])
+  
+    )
    action = "Testing tools"
    return_code = galaxy_run.run_galaxy_command(
        ctx,

diff --git a/planemo/galaxy_test/structures.py b/planemo/galaxy_test/structures.py
@@ -10,7 +10,7 @@
 from planemo.io import error
 RUN_TESTS_CMD = (
-  
-    "sh run_tests.sh --report_file %s %s %s %s"
+  
+    "sh run_tests.sh $COMMON_STARTUP_ARGS --report_file %s %s %s %s"
 )

diff --git a/planemo/io.py b/planemo/io.py
@@ -67,6 +67,10 @@ def _echo(message, err=False):
        print(message)
+  
+def shell_join(*args):
+  
+    return "; ".join([c for c in args if c])
+  
+
+  
+
 def write_file(path, content):
    with open(path, "w") as f:
        f.write(content)