Update planemo for Galaxy wheels.

jmchilton · jmchilton · commit 25ef0d569e94 · 2015-11-11T11:23:06.000Z
That got out of hand quickly.
diff --git a/planemo/galaxy_config.py b/planemo/galaxy_config.py
@@ -16,6 +16,7 @@
 from planemo import galaxy_run
 from planemo.io import warn
 from planemo.io import shell
+from planemo.io import shell_join
 from planemo.io import write_file
 from planemo.io import kill_pid_file
 from planemo.io import wait_on
@@ -92,7 +93,9 @@
 DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
 LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
 
-PIP_INSTALL_CMD = "[ -d .venv ] && . .venv/bin/activate && pip install %s"
+PIP_INSTALL_CMD = "pip install %s"
+
+COMMAND_STARTUP_COMMAND = "./scripts/common_startup.sh ${COMMON_STARTUP_ARGS}"
 
 FAILED_TO_FIND_GALAXY_EXCEPTION = (
     "Failed to find Galaxy root directory - please explicitly specify one "
@@ -523,21 +526,21 @@ def _install_galaxy(ctx, config_directory, kwds):
     if not kwds.get("no_cache_galaxy", False):
         _install_galaxy_via_git(ctx, config_directory, kwds)
     else:
-        _install_galaxy_via_download(config_directory, kwds)
+        _install_galaxy_via_download(ctx, config_directory, kwds)
 
 
-def _install_galaxy_via_download(config_directory, kwds):
+def _install_galaxy_via_download(ctx, config_directory, kwds):
     branch = _galaxy_branch(kwds)
     tar_cmd = "tar -zxvf %s" % branch
     command = galaxy_run.DOWNLOAD_GALAXY + "; %s | tail" % tar_cmd
-    _install_with_command(config_directory, command, kwds)
+    _install_with_command(ctx, config_directory, command, kwds)
 
 
 def _install_galaxy_via_git(ctx, config_directory, kwds):
     gx_repo = _ensure_galaxy_repository_available(ctx, kwds)
     branch = _galaxy_branch(kwds)
     command = git.command_clone(ctx, gx_repo, "galaxy-dev", branch=branch)
-    _install_with_command(config_directory, command, kwds)
+    _install_with_command(ctx, config_directory, command, kwds)
 
 
 def _build_eggs_cache(ctx, env, kwds):
@@ -557,7 +560,7 @@ def _galaxy_branch(kwds):
     return branch
 
 
-def _install_with_command(config_directory, command, kwds):
+def _install_with_command(ctx, config_directory, command, kwds):
     # TODO: --watchdog
     pip_installs = []
     if kwds.get("cwl", False):
@@ -566,15 +569,17 @@ def _install_with_command(config_directory, command, kwds):
         pip_install_command = PIP_INSTALL_CMD % " ".join(pip_installs)
     else:
         pip_install_command = ""
-    install_cmds = [
+    setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
+    install_cmd = shell_join(
         "cd %s" % config_directory,
         command,
         "cd galaxy-dev",
-        "type virtualenv >/dev/null 2>&1 && virtualenv .venv",
+        setup_venv_command,
         pip_install_command,
-        galaxy_run.ACTIVATE_COMMAND,
-    ]
-    shell(";".join([c for c in install_cmds if c]))
+        galaxy_run.setup_common_startup_args(),
+        COMMAND_STARTUP_COMMAND,
+    )
+    shell(install_cmd)
 
 
 def _ensure_galaxy_repository_available(ctx, kwds):
diff --git a/planemo/galaxy_run.py b/planemo/galaxy_run.py
@@ -1,16 +1,88 @@
-from planemo.io import info
+import os
+import string
+
+from planemo.io import info, shell_join
 from galaxy.tools.deps.commands import shell
 
 
 # Activate galaxy's virtualenv if present (needed for tests say but not for
 # server because run.sh does this).
-ACTIVATE_COMMAND = "[ -e .venv ] && . .venv/bin/activate"
+ACTIVATE_COMMAND = "[ -e $GALAXY_VIRTUAL_ENV ] && . $GALAXY_VIRTUAL_ENV/bin/activate"
+CREATE_COMMAND = shell_join(
+    'if [ ! -e $GALAXY_VIRTUAL_ENV ]',
+    ' then type virtualenv >/dev/null 2>&1 && virtualenv $GALAXY_VIRTUAL_ENV',
+    ' else echo "Reusing existing virtualenv $GALAXY_VIRTUAL_ENV"',
+    ' fi',
+)
+PRINT_VENV_COMMAND = shell_join(
+    'echo "Set \$GALAXY_VIRTUAL_ENV to $GALAXY_VIRTUAL_ENV"',
+    'if [ -e $GALAXY_VIRTUAL_ENV ]',
+    'then echo "Virtual environment directory exists."',
+    'else echo "Virtual environment directory does not exist."',
+    'fi',
+)
+
 
 # TODO: Mac-y curl variant of this.
 DOWNLOAD_GALAXY = (
     "wget https://codeload.github.com/galaxyproject/galaxy/tar.gz/dev"
 )
 
+CACHED_VIRTUAL_ENV_COMMAND = ("if [ -d .venv ] || [ -f dist-eggs.ini ];"
+                              " then GALAXY_VIRTUAL_ENV=.venv; "
+                              " else GALAXY_VIRTUAL_ENV=%s; fi")
+UNCACHED_VIRTUAL_ENV_COMMAND = "GALAXY_VIRTUAL_ENV=.venv"
+
+
+def setup_venv(ctx, kwds):
+    return shell_join(
+        locate_galaxy_virtualenv(ctx, kwds),
+        PRINT_VENV_COMMAND if ctx.verbose else None,
+        CREATE_COMMAND,
+        PRINT_VENV_COMMAND if ctx.verbose else None,
+        ACTIVATE_COMMAND,
+    )
+
+
+def locate_galaxy_virtualenv(ctx, kwds):
+    if not kwds.get("no_cache_galaxy", False):
+        workspace = ctx.workspace
+        shared_venv_path = os.path.join(workspace, "gx_venv")
+        venv_command = CACHED_VIRTUAL_ENV_COMMAND % shared_venv_path
+    else:
+        venv_command = UNCACHED_VIRTUAL_ENV_COMMAND
+    return shell_join(
+        venv_command,
+        "export GALAXY_VIRTUAL_ENV",
+    )
+
+
+def shell_if_wheels(command):
+    """ Take a shell command and convert it to shell command that runs
+    only if Galaxy is new enough to use wheels.
+    """
+    return "$(grep -q 'skip-venv' run_tests.sh) && %s" % command
+
+
+def setup_common_startup_args():
+    return set_variable_if_wheels(
+        "COMMON_STARTUP_ARGS", "--skip-venv --dev-wheels"
+    )
+
+
+def set_variable_if_wheels(var, if_wheels_val, else_val=""):
+    var_command = '${var}=${else_val}; '
+    var_command += shell_if_wheels(
+        '${var}="${if_wheels_val}"; '
+    )
+    var_command += "export ${var}"
+    var_command += '; echo "Set ${var} to ${${var}}"'
+    return string.Template(var_command).safe_substitute(
+        var=var,
+        if_wheels_val=if_wheels_val,
+        else_val=else_val,
+    )
+
 
 def run_galaxy_command(ctx, command, env, action, daemon=False):
     message = "%s with command [%s]" % (action, command)
diff --git a/planemo/galaxy_serve.py b/planemo/galaxy_serve.py
@@ -17,18 +17,26 @@ def serve(ctx, paths, **kwds):
 
     with galaxy_config.galaxy_config(ctx, paths, **kwds) as config:
         # TODO: Allow running dockerized Galaxy here instead.
+        setup_common_startup_args = galaxy_run.set_variable_if_wheels(
+            "COMMON_STARTUP_ARGS", "--skip-venv"
+        )
+        setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
         run_script = os.path.join(config.galaxy_root, "run.sh")
+        run_script += " $COMMON_STARTUP_ARGS"
         if daemon:
             run_script += " --daemon --wait"
             config.env["GALAXY_RUN_ALL"] = "1"
         else:
             run_script += " --server-name '%s' --reload" % config.server_name
         server_ini = os.path.join(config.config_directory, "galaxy.ini")
         config.env["GALAXY_CONFIG_FILE"] = server_ini
-        cmds = [
+        cd_to_galaxy_command = "cd %s" % config.galaxy_root
+        cmd = io.shell_join(
+            cd_to_galaxy_command,
+            setup_common_startup_args,
+            setup_venv_command,
             run_script,
-        ]
-        cmd = "; ".join(cmds)
+        )
         action = "Starting galaxy"
         galaxy_run.run_galaxy_command(
             ctx,
diff --git a/planemo/galaxy_test/actions.py b/planemo/galaxy_test/actions.py
@@ -3,7 +3,7 @@
 import click
 
 from . import structures as test_structures
-from planemo.io import info, warn
+from planemo.io import info, warn, shell_join
 from planemo import galaxy_run
 from planemo.reports import build_report
 
@@ -51,12 +51,16 @@ def run_in_config(ctx, config, **kwds):
         failed=kwds.get("failed", False),
         installed=kwds.get("installed", False),
     ).build()
-    cmd = "; ".join([
+    setup_common_startup_args = galaxy_run.set_variable_if_wheels(
+        "COMMON_STARTUP_ARGS", "--skip-venv --skip-common-startup"
+    )
+    setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
+    cmd = shell_join(
         cd_to_galaxy_command,
-        galaxy_run.ACTIVATE_COMMAND,  # TODO: this should be moved to
-                                      # run_tests.sh to match run.sh.
+        setup_common_startup_args,
+        setup_venv_command,
         test_cmd,
-    ])
+    )
     action = "Testing tools"
     return_code = galaxy_run.run_galaxy_command(
         ctx,
diff --git a/planemo/galaxy_test/structures.py b/planemo/galaxy_test/structures.py
@@ -10,7 +10,7 @@
 from planemo.io import error
 
 RUN_TESTS_CMD = (
-    "sh run_tests.sh --report_file %s %s %s %s"
+    "sh run_tests.sh $COMMON_STARTUP_ARGS --report_file %s %s %s %s"
 )
 
 
diff --git a/planemo/io.py b/planemo/io.py
@@ -67,6 +67,10 @@ def _echo(message, err=False):
         print(message)
 
 
+def shell_join(*args):
+    return "; ".join([c for c in args if c])
+
+
 def write_file(path, content):
     with open(path, "w") as f:
         f.write(content)

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@`
`10`	`10`	`from planemo.io import error`
`11`	`11`
`12`	`12`	`RUN_TESTS_CMD = (`
`13`		`- "sh run_tests.sh --report_file %s %s %s %s"`
	`13`	`+ "sh run_tests.sh $COMMON_STARTUP_ARGS --report_file %s %s %s %s"`
`14`	`14`	`)`
`15`	`15`
`16`	`16`