Implement Galaxy profiles.

Specify ``--profile <name>`` with serve, test, or their shed counterparts to maintain a Galaxy database and set of shed installs across invocations.
galaxyproject · Apr 12, 2016 · 5d08b6766d18d346ff14ede862f1bc703b09df73 · 5d08b67
1 parent 2052db0
commit 5d08b6766d18d346ff14ede862f1bc703b09df73
Unified Split

Showing with 121 additions and 16 deletions.

+21 −9 planemo/cli.py

+1 −1 planemo/config.py

+19 −4 planemo/galaxy/config.py

+43 −0 planemo/galaxy/profiles.py

+4 −1 planemo/io.py

+11 −0 planemo/options.py

+22 −1 tests/test_cmd_serve.py
diff --git a/planemo/cli.py b/planemo/cli.py
@@ -6,7 +6,11 @@
 import click
 from .io import error
-  
-from .config import read_global_config
+  
+from .config import (
+  
+    read_global_config,
+  
+    OptionSource,
+  
+)
+  
+from planemo.galaxy import profiles
 from planemo import __version__
 PYTHON_2_7_COMMANDS = ["cwl_run", "cwl_script"]
@@ -39,14 +43,15 @@ def __init__(self):
        self.planemo_directory = None
        self.option_source = {}
-  
-    def set_option_source(self, param_name, option_source):
+  
+    def set_option_source(self, param_name, option_source, force=False):
        """Specify how an option was set."""
-  
-        assert param_name not in self.option_source
+  
+        if not force:
+  
+            assert param_name not in self.option_source
        self.option_source[param_name] = option_source
    def get_option_source(self, param_name):
        """Return OptionSource value indicating how the option was set."""
-  
-        assert param_name not in self.option_source
+  
+        assert param_name in self.option_source
        return self.option_source[param_name]
    @property
@@ -143,11 +148,18 @@ def get_command(self, ctx, name):
 def command_function(f):
    """Extension point for processing kwds after click callbacks."""
-  
-    def outer(*args, **kwargs):
-  
-        # arg_spec = inspect.getargspec(f)
-  
-        return f(*args, **kwargs)
-  
-    outer.__doc__ = f.__doc__
-  
-    return pass_context(outer)
+  
+    def handle_profile_options(*args, **kwds):
+  
+        profile = kwds.get("profile", None)
+  
+        if profile:
+  
+            ctx = args[0]
+  
+            profile_defaults = profiles.ensure_profile(ctx, profile, **kwds)
+  
+            for key, value in profile_defaults.items():
+  
+                if ctx.get_option_source(key) != OptionSource.cli:
+  
+                    kwds[key] = value
+  
+                    ctx.set_option_source(key, OptionSource.profile, force=True)
+  
+        return f(*args, **kwds)
+  
+    handle_profile_options.__doc__ = f.__doc__
+  
+    return pass_context(handle_profile_options)
 @click.command(cls=PlanemoCLI, context_settings=CONTEXT_SETTINGS)

diff --git a/planemo/config.py b/planemo/config.py
@@ -13,7 +13,7 @@
 OptionSource = aenum.Enum(
-  
-    "OptionSource", 'cli global_config default'
+  
+    "OptionSource", 'cli profile global_config default'
 )

diff --git a/planemo/galaxy/config.py b/planemo/galaxy/config.py
@@ -155,13 +155,21 @@ def config_join(*args):
        _handle_dependency_resolution(config_directory, kwds)
        _handle_job_metrics(config_directory, kwds)
        file_path = kwds.get("file_path") or config_join("files")
+  
+        _ensure_directory(file_path)
+  
+
+  
+        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
+  
+        _ensure_directory(tool_dependency_dir)
+  
+
        shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
        tool_definition = _tool_conf_entry_for(tool_paths)
        empty_tool_conf = config_join("empty_tool_conf.xml")
        tool_conf = config_join("tool_conf.xml")
        database_location = config_join("galaxy.sqlite")
+  
+
        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
+  
+        _ensure_directory(shed_tool_path)
+  
+
        sheds_config_path = _configure_sheds_config_file(
            ctx, config_directory, **kwds
        )
@@ -173,7 +181,7 @@ def config_join(*args):
            latest_galaxy=latest_galaxy,
            **kwds
        )
-  
-        os.makedirs(shed_tool_path)
+  
+        _ensure_directory(shed_tool_path)
        server_name = "planemo%d" % random.randint(0, 100000)
        port = int(kwds.get("port", 9090))
        template_args = dict(
@@ -257,7 +265,8 @@ def config_join(*args):
        write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)
        shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
-  
-        write_file(shed_tool_conf, shed_tool_conf_contents)
+  
+        # Write a new shed_tool_conf.xml if needed.
+  
+        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)
        pid_file = kwds.get("pid_file") or config_join("galaxy.pid")
@@ -411,7 +420,7 @@ def _download_database_template(
        shutil.copyfile(galaxy_sqlite_database, database_location)
        return True
-  
-    if latest:
+  
+    if latest or not galaxy_root:
        template_url = DOWNLOADS_URL + urlopen(LATEST_URL).read()
        urlretrieve(template_url, database_location)
        return True
@@ -748,7 +757,6 @@ def _handle_kwd_overrides(properties, kwds):
        'job_config_file',
        'job_metrics_config_file',
        'dependency_resolvers_config_file',
-  
-        'tool_dependency_dir',
    ]
    for prop in kwds_gx_properties:
        val = kwds.get(prop, None)
@@ -761,7 +769,14 @@ def _sub(template, args):
        return ''
    return Template(template).safe_substitute(args)
+  
+
+  
+def _ensure_directory(path):
+  
+    if path is not None and not os.path.exists(path):
+  
+        os.makedirs(path)
+  
+
+  
+
 __all__ = [
    "attempt_database_preseed",
+  
+    "DATABASE_LOCATION_TEMPLATE",
    "galaxy_config",
 ]
diff --git a/planemo/galaxy/profiles.py b/planemo/galaxy/profiles.py
@@ -0,0 +1,43 @@
+  
+"""This modules describes the abstraction of a Galaxy profile.
+  
+
+  
+This is a workspace with a specific default configuration and shed
+  
+tool setup. It is meant to be used with various serve commands.
+  
+"""
+  
+import os
+  
+
+  
+from .config import (
+  
+    DATABASE_LOCATION_TEMPLATE,
+  
+    attempt_database_preseed,
+  
+)
+  
+
+  
+
+  
+def ensure_profile(ctx, profile_name, **kwds):
+  
+    """Ensure a Galaxy profile exists and return profile defaults."""
+  
+    profile_directory = _profile_directory(ctx, profile_name)
+  
+    database_location = os.path.join(profile_directory, "galaxy.sqlite")
+  
+
+  
+    if not os.path.exists(profile_directory):
+  
+        os.makedirs(profile_directory)
+  
+        attempt_database_preseed(None, database_location, **kwds)
+  
+
+  
+    database_connection = DATABASE_LOCATION_TEMPLATE % database_location
+  
+    file_path = os.path.join(profile_directory, "files")
+  
+    shed_tool_path = os.path.join(profile_directory, "shed_tools")
+  
+    shed_tool_conf = os.path.join(profile_directory, "shed_tool_conf.xml")
+  
+    tool_dependency_dir = os.path.join(profile_directory, "deps")
+  
+
+  
+    return dict(
+  
+        file_path=file_path,
+  
+        database_connection=database_connection,
+  
+        tool_dependency_dir=tool_dependency_dir,
+  
+        shed_tool_conf=shed_tool_conf,
+  
+        shed_tool_path=shed_tool_path,
+  
+    )
+  
+
+  
+
+  
+def _profile_directory(ctx, profile_name):
+  
+    return os.path.join(ctx.galaxy_profiles_directory, profile_name)
+  
+
+  
+__all__ = [
+  
+    "ensure_profile",
+  
+]
diff --git a/planemo/io.py b/planemo/io.py
@@ -75,7 +75,10 @@ def shell_join(*args):
    return "; ".join([c for c in args if c])
-  
-def write_file(path, content):
+  
+def write_file(path, content, force=True):
+  
+    if os.path.exists(path) and not force:
+  
+        return
+  
+
    with open(path, "w") as f:
        f.write(content)

diff --git a/planemo/options.py b/planemo/options.py
@@ -211,6 +211,7 @@ def tool_dependency_dir_option():
            dir_okay=True,
            resolve_path=True
        ),
+  
+        default=None,
        use_global_config=True,
        help="Tool dependency dir for Galaxy to target.",
    )
@@ -701,6 +702,7 @@ def galaxy_config_options():
        conda_auto_install_option(),
        conda_auto_init_option(),
        # Profile options...
+  
+        profile_option(),
        file_path_option(),
        database_connection_option(),
        shed_tools_conf_option(),
@@ -740,6 +742,15 @@ def daemon_option():
    )
+  
+def profile_option():
+  
+    return planemo_option(
+  
+        "--profile",
+  
+        type=str,
+  
+        default=None,
+  
+        help="Location of pid file is executed with --daemon."
+  
+    )
+  
+
+  
+
 def galaxy_serve_options():
    return _compose(
        galaxy_run_options(),

diff --git a/tests/test_cmd_serve.py b/tests/test_cmd_serve.py
@@ -5,6 +5,7 @@
 from planemo.galaxy import api
 from planemo import network_util
+  
+from planemo.io import kill_pid_file
 from .test_utils import (
    CliTestCase,
@@ -30,20 +31,40 @@ def test_serve_daemon(self):
        extra_args = ["--daemon", "--pid_file", pid_file]
        serve = functools.partial(self._run, port, extra_args)
        self._launch_thread_and_wait(serve, port)
-  
-        time.sleep(.1)
+  
+        admin_gi = api.gi(port)
+  
+        user_api_key = api.user_api_key(admin_gi)
+  
+        user_gi = api.gi(port, user_api_key)
+  
+        assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
+  
+        user_gi.histories.create_history(TEST_HISTORY_NAME)
+  
+
+  
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+  
+    def test_serve_profile(self):
+  
+        port = network_util.get_free_port()
+  
+        pid_file = os.path.join(self._home, "test.pid")
+  
+        extra_args = [
+  
+            "--daemon", "--pid_file", pid_file, "--profile", "moo",
+  
+        ]
+  
+        serve = functools.partial(self._run, port, extra_args)
+  
+        self._launch_thread_and_wait(serve, port)
        assert network_util.wait_net_service("127.0.0.1", port)
        admin_gi = api.gi(port)
        user_api_key = api.user_api_key(admin_gi)
        user_gi = api.gi(port, user_api_key)
        assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
        user_gi.histories.create_history(TEST_HISTORY_NAME)
+  
+        kill_pid_file(pid_file)
+  
+
+  
+        self._launch_thread_and_wait(serve, port)
+  
+        assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 1
    def _launch_thread_and_wait(self, func, port):
        t = threading.Thread(target=func)
        t.daemon = True
        t.start()
        time.sleep(15)
        assert network_util.wait_net_service("127.0.0.1", port)
+  
+        time.sleep(.1)
+  
+        assert network_util.wait_net_service("127.0.0.1", port)
    def _run(self, port, serve_args=[]):
        cat_path = os.path.join(TEST_REPOS_DIR, "single_tool", "cat.xml")