Implement Galaxy profiles.

Specify ``--profile <name>`` with serve, test, or their shed counterparts to maintain a Galaxy database and set of shed installs across invocations.

Author: jmchilton

planemo/cli.py

@@ -6,7 +6,11 @@
 import click
 
 from .io import error
-from .config import read_global_config
+from .config import (
+    read_global_config,
+    OptionSource,
+)
+from planemo.galaxy import profiles
 from planemo import __version__
 
 PYTHON_2_7_COMMANDS = ["cwl_run", "cwl_script"]
@@ -39,14 +43,15 @@ def __init__(self):
         self.planemo_directory = None
         self.option_source = {}
 
-    def set_option_source(self, param_name, option_source):
+    def set_option_source(self, param_name, option_source, force=False):
         """Specify how an option was set."""
-        assert param_name not in self.option_source
+        if not force:
+            assert param_name not in self.option_source
         self.option_source[param_name] = option_source
 
     def get_option_source(self, param_name):
         """Return OptionSource value indicating how the option was set."""
-        assert param_name not in self.option_source
+        assert param_name in self.option_source
         return self.option_source[param_name]
 
     @property
@@ -143,11 +148,18 @@ def get_command(self, ctx, name):
 
 def command_function(f):
     """Extension point for processing kwds after click callbacks."""
-    def outer(*args, **kwargs):
-        # arg_spec = inspect.getargspec(f)
-        return f(*args, **kwargs)
-    outer.__doc__ = f.__doc__
-    return pass_context(outer)
+    def handle_profile_options(*args, **kwds):
+        profile = kwds.get("profile", None)
+        if profile:
+            ctx = args[0]
+            profile_defaults = profiles.ensure_profile(ctx, profile, **kwds)
+            for key, value in profile_defaults.items():
+                if ctx.get_option_source(key) != OptionSource.cli:
+                    kwds[key] = value
+                    ctx.set_option_source(key, OptionSource.profile, force=True)
+        return f(*args, **kwds)
+    handle_profile_options.__doc__ = f.__doc__
+    return pass_context(handle_profile_options)
 
 
 @click.command(cls=PlanemoCLI, context_settings=CONTEXT_SETTINGS)

planemo/config.py

@@ -13,7 +13,7 @@
 
 
 OptionSource = aenum.Enum(
-    "OptionSource", 'cli global_config default'
+    "OptionSource", 'cli profile global_config default'
 )
 
 

planemo/galaxy/config.py

@@ -155,13 +155,21 @@ def config_join(*args):
         _handle_dependency_resolution(config_directory, kwds)
         _handle_job_metrics(config_directory, kwds)
         file_path = kwds.get("file_path") or config_join("files")
+        _ensure_directory(file_path)
+
+        tool_dependency_dir = kwds.get("tool_dependency_dir") or config_join("deps")
+        _ensure_directory(tool_dependency_dir)
+
         shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
         tool_definition = _tool_conf_entry_for(tool_paths)
         empty_tool_conf = config_join("empty_tool_conf.xml")
 
         tool_conf = config_join("tool_conf.xml")
         database_location = config_join("galaxy.sqlite")
+
         shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
+        _ensure_directory(shed_tool_path)
+
         sheds_config_path = _configure_sheds_config_file(
             ctx, config_directory, **kwds
         )
@@ -173,7 +181,7 @@ def config_join(*args):
             latest_galaxy=latest_galaxy,
             **kwds
         )
-        os.makedirs(shed_tool_path)
+        _ensure_directory(shed_tool_path)
         server_name = "planemo%d" % random.randint(0, 100000)
         port = int(kwds.get("port", 9090))
         template_args = dict(
@@ -257,7 +265,8 @@ def config_join(*args):
         write_file(empty_tool_conf, EMPTY_TOOL_CONF_TEMPLATE)
 
         shed_tool_conf_contents = _sub(SHED_TOOL_CONF_TEMPLATE, template_args)
-        write_file(shed_tool_conf, shed_tool_conf_contents)
+        # Write a new shed_tool_conf.xml if needed.
+        write_file(shed_tool_conf, shed_tool_conf_contents, force=False)
 
         pid_file = kwds.get("pid_file") or config_join("galaxy.pid")
 
@@ -411,7 +420,7 @@ def _download_database_template(
         shutil.copyfile(galaxy_sqlite_database, database_location)
         return True
 
-    if latest:
+    if latest or not galaxy_root:
         template_url = DOWNLOADS_URL + urlopen(LATEST_URL).read()
         urlretrieve(template_url, database_location)
         return True
@@ -748,7 +757,6 @@ def _handle_kwd_overrides(properties, kwds):
         'job_config_file',
         'job_metrics_config_file',
         'dependency_resolvers_config_file',
-        'tool_dependency_dir',
     ]
     for prop in kwds_gx_properties:
         val = kwds.get(prop, None)
@@ -761,7 +769,14 @@ def _sub(template, args):
         return ''
     return Template(template).safe_substitute(args)
 
+
+def _ensure_directory(path):
+    if path is not None and not os.path.exists(path):
+        os.makedirs(path)
+
+
 __all__ = [
     "attempt_database_preseed",
+    "DATABASE_LOCATION_TEMPLATE",
     "galaxy_config",
 ]

planemo/galaxy/profiles.py

@@ -0,0 +1,43 @@
+"""This modules describes the abstraction of a Galaxy profile.
+
+This is a workspace with a specific default configuration and shed
+tool setup. It is meant to be used with various serve commands.
+"""
+import os
+
+from .config import (
+    DATABASE_LOCATION_TEMPLATE,
+    attempt_database_preseed,
+)
+
+
+def ensure_profile(ctx, profile_name, **kwds):
+    """Ensure a Galaxy profile exists and return profile defaults."""
+    profile_directory = _profile_directory(ctx, profile_name)
+    database_location = os.path.join(profile_directory, "galaxy.sqlite")
+
+    if not os.path.exists(profile_directory):
+        os.makedirs(profile_directory)
+        attempt_database_preseed(None, database_location, **kwds)
+
+    database_connection = DATABASE_LOCATION_TEMPLATE % database_location
+    file_path = os.path.join(profile_directory, "files")
+    shed_tool_path = os.path.join(profile_directory, "shed_tools")
+    shed_tool_conf = os.path.join(profile_directory, "shed_tool_conf.xml")
+    tool_dependency_dir = os.path.join(profile_directory, "deps")
+
+    return dict(
+        file_path=file_path,
+        database_connection=database_connection,
+        tool_dependency_dir=tool_dependency_dir,
+        shed_tool_conf=shed_tool_conf,
+        shed_tool_path=shed_tool_path,
+    )
+
+
+def _profile_directory(ctx, profile_name):
+    return os.path.join(ctx.galaxy_profiles_directory, profile_name)
+
+__all__ = [
+    "ensure_profile",
+]

planemo/io.py

@@ -75,7 +75,10 @@ def shell_join(*args):
     return "; ".join([c for c in args if c])
 
 
-def write_file(path, content):
+def write_file(path, content, force=True):
+    if os.path.exists(path) and not force:
+        return
+
     with open(path, "w") as f:
         f.write(content)
 

planemo/options.py

@@ -211,6 +211,7 @@ def tool_dependency_dir_option():
             dir_okay=True,
             resolve_path=True
         ),
+        default=None,
         use_global_config=True,
         help="Tool dependency dir for Galaxy to target.",
     )
@@ -701,6 +702,7 @@ def galaxy_config_options():
         conda_auto_install_option(),
         conda_auto_init_option(),
         # Profile options...
+        profile_option(),
         file_path_option(),
         database_connection_option(),
         shed_tools_conf_option(),
@@ -740,6 +742,15 @@ def daemon_option():
     )
 
 
+def profile_option():
+    return planemo_option(
+        "--profile",
+        type=str,
+        default=None,
+        help="Location of pid file is executed with --daemon."
+    )
+
+
 def galaxy_serve_options():
     return _compose(
         galaxy_run_options(),

tests/test_cmd_serve.py

@@ -5,6 +5,7 @@
 
 from planemo.galaxy import api
 from planemo import network_util
+from planemo.io import kill_pid_file
 
 from .test_utils import (
     CliTestCase,
@@ -30,20 +31,40 @@ def test_serve_daemon(self):
         extra_args = ["--daemon", "--pid_file", pid_file]
         serve = functools.partial(self._run, port, extra_args)
         self._launch_thread_and_wait(serve, port)
-        time.sleep(.1)
+        admin_gi = api.gi(port)
+        user_api_key = api.user_api_key(admin_gi)
+        user_gi = api.gi(port, user_api_key)
+        assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
+        user_gi.histories.create_history(TEST_HISTORY_NAME)
+
+    @skip_if_environ("PLANEMO_SKIP_GALAXY_TESTS")
+    def test_serve_profile(self):
+        port = network_util.get_free_port()
+        pid_file = os.path.join(self._home, "test.pid")
+        extra_args = [
+            "--daemon", "--pid_file", pid_file, "--profile", "moo",
+        ]
+        serve = functools.partial(self._run, port, extra_args)
+        self._launch_thread_and_wait(serve, port)
         assert network_util.wait_net_service("127.0.0.1", port)
         admin_gi = api.gi(port)
         user_api_key = api.user_api_key(admin_gi)
         user_gi = api.gi(port, user_api_key)
         assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 0
         user_gi.histories.create_history(TEST_HISTORY_NAME)
+        kill_pid_file(pid_file)
+
+        self._launch_thread_and_wait(serve, port)
+        assert len(user_gi.histories.get_histories(name=TEST_HISTORY_NAME)) == 1
 
     def _launch_thread_and_wait(self, func, port):
         t = threading.Thread(target=func)
         t.daemon = True
         t.start()
         time.sleep(15)
         assert network_util.wait_net_service("127.0.0.1", port)
+        time.sleep(.1)
+        assert network_util.wait_net_service("127.0.0.1", port)
 
     def _run(self, port, serve_args=[]):
         cat_path = os.path.join(TEST_REPOS_DIR, "single_tool", "cat.xml")