Permalink
Browse files

Implement Galaxy profiles.

Specify ``--profile <name>`` with serve, test, or their shed counterparts to maintain a Galaxy database and set of shed installs across invocations.
  • Loading branch information...
jmchilton committed Apr 12, 2016
1 parent 2052db0 commit 5d08b6766d18d346ff14ede862f1bc703b09df73
Showing with 121 additions and 16 deletions.
  1. +21 −9 planemo/cli.py
  2. +1 −1 planemo/config.py
  3. +19 −4 planemo/galaxy/config.py
  4. +43 −0 planemo/galaxy/profiles.py
  5. +4 −1 planemo/io.py
  6. +11 −0 planemo/options.py
  7. +22 −1 tests/test_cmd_serve.py
@@ -6,7 +6,11 @@
import click

from .io import error
from .config import read_global_config
from .config import (
read_global_config,
OptionSource,
)
from planemo.galaxy import profiles
from planemo import __version__

PYTHON_2_7_COMMANDS = ["cwl_run", "cwl_script"]
@@ -39,14 +43,15 @@ def __init__(self):
self.planemo_directory = None
self.option_source = {}

def set_option_source(self, param_name, option_source):
def set_option_source(self, param_name, option_source, force=False):
"""Specify how an option was set."""
assert param_name not in self.option_source
if not force:
assert param_name not in self.option_source
self.option_source[param_name] = option_source

def get_option_source(self, param_name):
"""Return OptionSource value indicating how the option was set."""
assert param_name not in self.option_source
assert param_name in self.option_source
return self.option_source[param_name]

@property
@@ -143,11 +148,18 @@ def get_command(self, ctx, name):

def command_function(f):
"""Extension point for processing kwds after click callbacks."""
def outer(*args, **kwargs):
# arg_spec = inspect.getargspec(f)
return f(*args, **kwargs)
outer.__doc__ = f.__doc__
return pass_context(outer)
def handle_profile_options(*args, **kwds):
profile = kwds.get("profile", None)
if profile:
ctx = args[0]
profile_defaults = profiles.ensure_profile(ctx, profile, **kwds)
for key, value in profile_defaults.items():
if ctx.get_option_source(key) != OptionSource.cli:
kwds[key] = value
ctx.set_option_source(key, OptionSource.profile, force=True)
return f(*args, **kwds)
handle_profile_options.__doc__ = f.__doc__
return pass_context(handle_profile_options)


@click.command(cls=PlanemoCLI, context_settings=CONTEXT_SETTINGS)
@@ -13,7 +13,7 @@


OptionSource = aenum.Enum(
"OptionSource", 'cli global_config default'
"OptionSource", 'cli profile global_config default'
)


@@ -155,13 +155,21 @@ def config_join(*args):
_handle_dependency_resolution(config_directory, kwds)
_handle_job_metrics(config_directory, kwds)
file_path = kwds.get("file_path") or config_join("files")
_ensure_directory(file_path)

tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
_ensure_directory(tool_dependency_dir)

shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
tool_definition = _tool_conf_entry_for(tool_paths)
empty_tool_conf = config_join("empty_tool_conf.xml")

tool_conf = config_join("tool_conf.xml")
database_location = config_join("galaxy.sqlite")

shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
_ensure_directory(shed_tool_path)

sheds_config_path = _configure_sheds_config_file(
ctx, config_directory, **kwds
)
@@ -173,7 +181,7 @@ def config_join(*args):
latest_galaxy=latest_galaxy,
**kwds
)
os.makedirs(shed_tool_path)
_ensure_directory(shed_tool_path)
server_name = "planemo%d" % random.randint(0, 100000)
port = int(kwds.get("port", 9090))
template_args = dict(
@@ -257,7 +265,8 @@ def config_join(*args):
write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)

shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
write_file(shed_tool_conf, shed_tool_conf_contents)
# Write a new shed_tool_conf.xml if needed.
write_file(shed_tool_conf, shed_tool_conf_contents, force=False)

pid_file = kwds.get("pid_file") or config_join("galaxy.pid")

@@ -411,7 +420,7 @@ def _download_database_template(
shutil.copyfile(galaxy_sqlite_database, database_location)
return True

if latest:
if latest or not galaxy_root:
template_url = DOWNLOADS_URL + urlopen(LATEST_URL).read()
urlretrieve(template_url, database_location)
return True
@@ -748,7 +757,6 @@ def _handle_kwd_overrides(properties, kwds):
'job_config_file',
'job_metrics_config_file',
'dependency_resolvers_config_file',
'tool_dependency_dir',
]
for prop in kwds_gx_properties:
val = kwds.get(prop, None)
@@ -761,7 +769,14 @@ def _sub(template, args):
return ''
return Template(template).safe_substitute(args)


def _ensure_directory(path):
if path is not None and not os.path.exists(path):
os.makedirs(path)


__all__ = [
"attempt_database_preseed",
"DATABASE_LOCATION_TEMPLATE",
"galaxy_config",
]
@@ -0,0 +1,43 @@
"""This modules describes the abstraction of a Galaxy profile.
This is a workspace with a specific default configuration and shed
tool setup. It is meant to be used with various serve commands.
"""
import os

from .config import (
DATABASE_LOCATION_TEMPLATE,
attempt_database_preseed,
)


def ensure_profile(ctx, profile_name, **kwds):
"""Ensure a Galaxy profile exists and return profile defaults."""
profile_directory = _profile_directory(ctx, profile_name)
database_location = os.path.join(profile_directory, "galaxy.sqlite")

if not os.path.exists(profile_directory):
os.makedirs(profile_directory)
attempt_database_preseed(None, database_location, **kwds)

database_connection = DATABASE_LOCATION_TEMPLATE % database_location
file_path = os.path.join(profile_directory, "files")
shed_tool_path = os.path.join(profile_directory, "shed_tools")
shed_tool_conf = os.path.join(profile_directory, "shed_tool_conf.xml")
tool_dependency_dir = os.path.join(profile_directory, "deps")

return dict(
file_path=file_path,
database_connection=database_connection,
tool_dependency_dir=tool_dependency_dir,
shed_tool_conf=shed_tool_conf,
shed_tool_path=shed_tool_path,
)


def _profile_directory(ctx, profile_name):
return os.path.join(ctx.galaxy_profiles_directory, profile_name)

__all__ = [
"ensure_profile",
]
@@ -75,7 +75,10 @@ def shell_join(*args):
return "; ".join([c for c in args if c])


def write_file(path, content):
def write_file(path, content, force=True):
if os.path.exists(path) and not force:
return

with open(path, "w") as f:
f.write(content)

@@ -211,6 +211,7 @@ def tool_dependency_dir_option():
dir_okay=True,
resolve_path=True
),
default=None,
use_global_config=True,
help="Tool dependency dir for Galaxy to target.",
)
@@ -701,6 +702,7 @@ def galaxy_config_options():
conda_auto_install_option(),
conda_auto_init_option(),
# Profile options...
profile_option(),
file_path_option(),
database_connection_option(),
shed_tools_conf_option(),
@@ -740,6 +742,15 @@ def daemon_option():
)


def profile_option():
return planemo_option(
"--profile",
type=str,
default=None,
help="Location of pid file is executed with --daemon."
)


def galaxy_serve_options():
return _compose(
galaxy_run_options(),
@@ -5,6 +5,7 @@

from planemo.galaxy import api
from planemo import network_util
from planemo.io import kill_pid_file

from .test_utils import (
CliTestCase,
@@ -30,20 +31,40 @@ def test_serve_daemon(self):
extra_args = ["--daemon", "--pid_file", pid_file]
serve = functools.partial(self._run, port, extra_args)
self._launch_thread_and_wait(serve, port)
time.sleep(.1)
admin_gi = api.gi(port)
user_api_key = api.user_api_key(admin_gi)
user_gi = api.gi(port, user_api_key)
assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
user_gi.histories.create_history(TEST_HISTORY_NAME)

@skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
def test_serve_profile(self):
port = network_util.get_free_port()
pid_file = os.path.join(self._home, "test.pid")
extra_args = [
"--daemon", "--pid_file", pid_file, "--profile", "moo",
]
serve = functools.partial(self._run, port, extra_args)
self._launch_thread_and_wait(serve, port)
assert network_util.wait_net_service("127.0.0.1", port)
admin_gi = api.gi(port)
user_api_key = api.user_api_key(admin_gi)
user_gi = api.gi(port, user_api_key)
assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
user_gi.histories.create_history(TEST_HISTORY_NAME)
kill_pid_file(pid_file)

self._launch_thread_and_wait(serve, port)
assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 1

def _launch_thread_and_wait(self, func, port):
t = threading.Thread(target=func)
t.daemon = True
t.start()
time.sleep(15)
assert network_util.wait_net_service("127.0.0.1", port)
time.sleep(.1)
assert network_util.wait_net_service("127.0.0.1", port)

def _run(self, port, serve_args=[]):
cat_path = os.path.join(TEST_REPOS_DIR, "single_tool", "cat.xml")

0 comments on commit 5d08b67

Please sign in to comment.