Permalink
Browse files

Update planemo for Galaxy wheels.

That got out of hand quickly.
  • Loading branch information...
jmchilton committed Nov 11, 2015
1 parent cbc2806 commit 25ef0d569e94c4d9f03e6f1225805eef6b65d9b5
Showing with 115 additions and 22 deletions.
  1. +16 −11 planemo/galaxy_config.py
  2. +74 −2 planemo/galaxy_run.py
  3. +11 −3 planemo/galaxy_serve.py
  4. +9 −5 planemo/galaxy_test/actions.py
  5. +1 −1 planemo/galaxy_test/structures.py
  6. +4 −0 planemo/io.py
@@ -16,6 +16,7 @@
from planemo import galaxy_run
from planemo.io import warn
from planemo.io import shell
from planemo.io import shell_join
from planemo.io import write_file
from planemo.io import kill_pid_file
from planemo.io import wait_on
@@ -92,7 +93,9 @@
DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
LATEST_URL = DOWNLOADS_URL + "latest.sqlite"

PIP_INSTALL_CMD = "[ -d .venv ] && . .venv/bin/activate && pip install %s"
PIP_INSTALL_CMD = "pip install %s"

COMMAND_STARTUP_COMMAND = "./scripts/common_startup.sh ${COMMON_STARTUP_ARGS}"

FAILED_TO_FIND_GALAXY_EXCEPTION = (
"Failed to find Galaxy root directory - please explicitly specify one "
@@ -523,21 +526,21 @@ def _install_galaxy(ctx, config_directory, kwds):
if not kwds.get("no_cache_galaxy", False):
_install_galaxy_via_git(ctx, config_directory, kwds)
else:
_install_galaxy_via_download(config_directory, kwds)
_install_galaxy_via_download(ctx, config_directory, kwds)


def _install_galaxy_via_download(config_directory, kwds):
def _install_galaxy_via_download(ctx, config_directory, kwds):
branch = _galaxy_branch(kwds)
tar_cmd = "tar -zxvf %s" % branch
command = galaxy_run.DOWNLOAD_GALAXY + "; %s | tail" % tar_cmd
_install_with_command(config_directory, command, kwds)
_install_with_command(ctx, config_directory, command, kwds)


def _install_galaxy_via_git(ctx, config_directory, kwds):
gx_repo = _ensure_galaxy_repository_available(ctx, kwds)
branch = _galaxy_branch(kwds)
command = git.command_clone(ctx, gx_repo, "galaxy-dev", branch=branch)
_install_with_command(config_directory, command, kwds)
_install_with_command(ctx, config_directory, command, kwds)


def _build_eggs_cache(ctx, env, kwds):
@@ -557,7 +560,7 @@ def _galaxy_branch(kwds):
return branch


def _install_with_command(config_directory, command, kwds):
def _install_with_command(ctx, config_directory, command, kwds):
# TODO: --watchdog
pip_installs = []
if kwds.get("cwl", False):
@@ -566,15 +569,17 @@ def _install_with_command(config_directory, command, kwds):
pip_install_command = PIP_INSTALL_CMD % " ".join(pip_installs)
else:
pip_install_command = ""
install_cmds = [
setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
install_cmd = shell_join(
"cd %s" % config_directory,
command,
"cd galaxy-dev",
"type virtualenv >/dev/null 2>&1 && virtualenv .venv",
setup_venv_command,
pip_install_command,
galaxy_run.ACTIVATE_COMMAND,
]
shell(";".join([c for c in install_cmds if c]))
galaxy_run.setup_common_startup_args(),
COMMAND_STARTUP_COMMAND,
)
shell(install_cmd)


def _ensure_galaxy_repository_available(ctx, kwds):
@@ -1,16 +1,88 @@
from planemo.io import info
import os
import string

from planemo.io import info, shell_join
from galaxy.tools.deps.commands import shell


# Activate galaxy's virtualenv if present (needed for tests say but not for
# server because run.sh does this).
ACTIVATE_COMMAND = "[ -e .venv ] && . .venv/bin/activate"
ACTIVATE_COMMAND = "[ -e $GALAXY_VIRTUAL_ENV ] && . $GALAXY_VIRTUAL_ENV/bin/activate"
CREATE_COMMAND = shell_join(
'if [ ! -e $GALAXY_VIRTUAL_ENV ]',
' then type virtualenv >/dev/null 2>&1 && virtualenv $GALAXY_VIRTUAL_ENV',
' else echo "Reusing existing virtualenv $GALAXY_VIRTUAL_ENV"',
' fi',
)
PRINT_VENV_COMMAND = shell_join(
'echo "Set \$GALAXY_VIRTUAL_ENV to $GALAXY_VIRTUAL_ENV"',
'if [ -e $GALAXY_VIRTUAL_ENV ]',
'then echo "Virtual environment directory exists."',
'else echo "Virtual environment directory does not exist."',
'fi',
)


# TODO: Mac-y curl variant of this.
DOWNLOAD_GALAXY = (
"wget https://codeload.github.com/galaxyproject/galaxy/tar.gz/dev"
)

CACHED_VIRTUAL_ENV_COMMAND = ("if [ -d .venv ] || [ -f dist-eggs.ini ];"
" then GALAXY_VIRTUAL_ENV=.venv; "
" else GALAXY_VIRTUAL_ENV=%s; fi")
UNCACHED_VIRTUAL_ENV_COMMAND = "GALAXY_VIRTUAL_ENV=.venv"


def setup_venv(ctx, kwds):
return shell_join(
locate_galaxy_virtualenv(ctx, kwds),
PRINT_VENV_COMMAND if ctx.verbose else None,
CREATE_COMMAND,
PRINT_VENV_COMMAND if ctx.verbose else None,
ACTIVATE_COMMAND,
)


def locate_galaxy_virtualenv(ctx, kwds):
if not kwds.get("no_cache_galaxy", False):
workspace = ctx.workspace
shared_venv_path = os.path.join(workspace, "gx_venv")
venv_command = CACHED_VIRTUAL_ENV_COMMAND % shared_venv_path
else:
venv_command = UNCACHED_VIRTUAL_ENV_COMMAND
return shell_join(
venv_command,
"export GALAXY_VIRTUAL_ENV",
)


def shell_if_wheels(command):
""" Take a shell command and convert it to shell command that runs
only if Galaxy is new enough to use wheels.
"""
return "$(grep -q 'skip-venv' run_tests.sh) && %s" % command


def setup_common_startup_args():
return set_variable_if_wheels(
"COMMON_STARTUP_ARGS", "--skip-venv --dev-wheels"
)


def set_variable_if_wheels(var, if_wheels_val, else_val=""):
var_command = '${var}=${else_val}; '
var_command += shell_if_wheels(
'${var}="${if_wheels_val}"; '
)
var_command += "export ${var}"
var_command += '; echo "Set ${var} to ${${var}}"'
return string.Template(var_command).safe_substitute(
var=var,
if_wheels_val=if_wheels_val,
else_val=else_val,
)


def run_galaxy_command(ctx, command, env, action, daemon=False):
message = "%s with command [%s]" % (action, command)
@@ -17,18 +17,26 @@ def serve(ctx, paths, **kwds):

with galaxy_config.galaxy_config(ctx, paths, **kwds) as config:
# TODO: Allow running dockerized Galaxy here instead.
setup_common_startup_args = galaxy_run.set_variable_if_wheels(
"COMMON_STARTUP_ARGS", "--skip-venv"
)
setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
run_script = os.path.join(config.galaxy_root, "run.sh")
run_script += " $COMMON_STARTUP_ARGS"
if daemon:
run_script += " --daemon --wait"
config.env["GALAXY_RUN_ALL"] = "1"
else:
run_script += " --server-name '%s' --reload" % config.server_name
server_ini = os.path.join(config.config_directory, "galaxy.ini")
config.env["GALAXY_CONFIG_FILE"] = server_ini
cmds = [
cd_to_galaxy_command = "cd %s" % config.galaxy_root
cmd = io.shell_join(
cd_to_galaxy_command,
setup_common_startup_args,
setup_venv_command,
run_script,
]
cmd = "; ".join(cmds)
)
action = "Starting galaxy"
galaxy_run.run_galaxy_command(
ctx,
@@ -3,7 +3,7 @@
import click

from . import structures as test_structures
from planemo.io import info, warn
from planemo.io import info, warn, shell_join
from planemo import galaxy_run
from planemo.reports import build_report

@@ -51,12 +51,16 @@ def run_in_config(ctx, config, **kwds):
failed=kwds.get("failed", False),
installed=kwds.get("installed", False),
).build()
cmd = "; ".join([
setup_common_startup_args = galaxy_run.set_variable_if_wheels(
"COMMON_STARTUP_ARGS", "--skip-venv --skip-common-startup"
)
setup_venv_command = galaxy_run.setup_venv(ctx, kwds)
cmd = shell_join(
cd_to_galaxy_command,
galaxy_run.ACTIVATE_COMMAND, # TODO: this should be moved to
# run_tests.sh to match run.sh.
setup_common_startup_args,
setup_venv_command,
test_cmd,
])
)
action = "Testing tools"
return_code = galaxy_run.run_galaxy_command(
ctx,
@@ -10,7 +10,7 @@
from planemo.io import error

RUN_TESTS_CMD = (
"sh run_tests.sh --report_file %s %s %s %s"
"sh run_tests.sh $COMMON_STARTUP_ARGS --report_file %s %s %s %s"
)


@@ -67,6 +67,10 @@ def _echo(message, err=False):
print(message)


def shell_join(*args):
return "; ".join([c for c in args if c])


def write_file(path, content):
with open(path, "w") as f:
f.write(content)

0 comments on commit 25ef0d5

Please sign in to comment.