Implement Galaxy options required for persistent serve environment.

Namely --file_path, --database_connection, --shed_tool_conf, --shed_tool_path. Each default-able in ~/.planemo.yml.

Add some initial and basic tests for ``planemo.galaxy.config`` and rework pre-seeding database ahead of implementing profiles for the above options.

planemo/cli.py

@@ -81,13 +81,22 @@ def workspace(self):
         if not self.planemo_directory:
             raise Exception("No planemo workspace defined.")
         workspace = self.planemo_directory
-        if not os.path.exists(workspace):
-            os.makedirs(workspace)
-        if not os.path.isdir(workspace):
-            template = "Planemo workspace directory [%s] unavailable."
-            message = template % workspace
+        return self._ensure_directory(workspace, "workspace")
+
+    @property
+    def galaxy_profiles_directory(self):
+        """Create a return a directory for storing Galaxy profiles."""
+        path = os.path.join(self.workspace, "profiles")
+        return self._ensure_directory(path, "Galaxy profiles")
+
+    def _ensure_directory(self, path, name):
+        if not os.path.exists(path):
+            os.makedirs(path)
+        if not os.path.isdir(path):
+            template = "Planemo %s directory [%s] unavailable."
+            message = template % (name, path)
             raise Exception(message)
-        return workspace
+        return path
 
 
 pass_context = click.make_pass_decorator(Context, ensure=True)

planemo/galaxy/config.py

@@ -57,7 +57,7 @@
 """
 
 SHED_TOOL_CONF_TEMPLATE = """<?xml version="1.0"?>
-<toolbox tool_path="${shed_tools_path}">
+<toolbox tool_path="${shed_tool_path}">
 </toolbox>
 """
 
@@ -111,6 +111,8 @@
 DOWNLOADABLE_MIGRATION_VERSIONS = [127, 120, 117]
 LATEST_URL = DOWNLOADS_URL + "latest.sqlite"
 
+DATABASE_LOCATION_TEMPLATE = "sqlite:///%s?isolation_level=IMMEDIATE"
+
 PIP_INSTALL_CMD = "pip install %s"
 
 COMMAND_STARTUP_COMMAND = "./scripts/common_startup.sh ${COMMON_STARTUP_ARGS}"
@@ -123,6 +125,7 @@
 
 @contextlib.contextmanager
 def galaxy_config(ctx, tool_paths, for_tests=False, **kwds):
+    """Set up a ``GalaxyConfig`` in an auto-cleaned context."""
     test_data_dir = _find_test_data(tool_paths, **kwds)
     tool_data_table = _find_tool_data_table(
         tool_paths,
@@ -151,12 +154,14 @@ def config_join(*args):
 
         _handle_dependency_resolution(config_directory, kwds)
         _handle_job_metrics(config_directory, kwds)
+        file_path = kwds.get("file_path") or config_join("files")
+        shed_tool_conf = kwds.get("shed_tool_conf") or config_join("shed_tools_conf.xml")
         tool_definition = _tool_conf_entry_for(tool_paths)
         empty_tool_conf = config_join("empty_tool_conf.xml")
-        shed_tool_conf = _shed_tool_conf(install_galaxy, config_directory)
+
         tool_conf = config_join("tool_conf.xml")
         database_location = config_join("galaxy.sqlite")
-        shed_tools_path = config_join("shed_tools")
+        shed_tool_path = kwds.get("shed_tool_path") or config_join("shed_tools")
         sheds_config_path = _configure_sheds_config_file(
             ctx, config_directory, **kwds
         )
@@ -168,15 +173,15 @@ def config_join(*args):
             latest_galaxy=latest_galaxy,
             **kwds
         )
-        os.makedirs(shed_tools_path)
+        os.makedirs(shed_tool_path)
         server_name = "planemo%d" % random.randint(0, 100000)
         port = int(kwds.get("port", 9090))
         template_args = dict(
             port=port,
             host=kwds.get("host", "127.0.0.1"),
             server_name=server_name,
             temp_directory=config_directory,
-            shed_tools_path=shed_tools_path,
+            shed_tool_path=shed_tool_path,
             database_location=database_location,
             tool_definition=tool_definition,
             tool_conf=tool_conf,
@@ -188,7 +193,7 @@ def config_join(*args):
         tool_config_file = "%s,%s" % (tool_conf, shed_tool_conf)
         properties = dict(
             tool_dependency_dir=dependency_dir,
-            file_path="${temp_directory}/files",
+            file_path=file_path,
             new_file_path="${temp_directory}/tmp",
             tool_config_file=tool_config_file,
             tool_sheds_config_file=sheds_config_path,
@@ -219,8 +224,7 @@ def config_join(*args):
             test_data_dir=test_data_dir,  # TODO: make gx respect this
         )
         if not for_tests:
-            properties["database_connection"] = \
-                "sqlite:///${database_location}?isolation_level=IMMEDIATE"
+            properties["database_connection"] = _database_connection(database_location, **kwds)
 
         _handle_kwd_overrides(properties, kwds)
 
@@ -363,9 +367,24 @@ def cleanup(self):
         shutil.rmtree(self.config_directory)
 
 
+def _database_connection(database_location, **kwds):
+    default_connection = DATABASE_LOCATION_TEMPLATE % database_location
+    database_connection = kwds.get("database_connection") or default_connection
+    return database_connection
+
+
 def attempt_database_preseed(
     effective_galaxy_root, database_location, latest_galaxy=False, **kwds
 ):
+    """If database location is unset, attempt to seed the database."""
+    if os.path.exists(database_location):
+        # Can't seed an existing database.
+        return False
+
+    if not _database_connection(database_location, **kwds).startswith("sqlite"):
+        # Not going to use an sqlite database, don't preseed.
+        return False
+
     preseeded_database = True
     galaxy_sqlite_database = kwds.get("galaxy_database_seed", None)
     try:
@@ -742,4 +761,7 @@ def _sub(template, args):
         return ''
     return Template(template).safe_substitute(args)
 
-__all__ = ["galaxy_config"]
+__all__ = [
+    "attempt_database_preseed",
+    "galaxy_config",
+]

planemo/options.py

@@ -158,16 +158,45 @@ def shed_dependency_resolution():
     )
 
 
-def job_config_option():
+def file_path_option():
     return planemo_option(
-        "--job_config_file",
+        "--file_path",
         type=click.Path(
-            exists=True,
-            file_okay=True,
-            dir_okay=False,
+            file_okay=False,
+            dir_okay=True,
             resolve_path=True
         ),
-        help="Job configuration file for Galaxy to target.",
+        help="Location for files created by Galaxy (e.g. database/files).",
+        default=None,
+        use_global_config=True,
+    )
+
+
+def database_connection_option():
+    return planemo_option(
+        "--database_connection",
+        type=str,
+        help="Database connection string to use for Galaxy.",
+        default=None,
+        use_global_config=True,
+    )
+
+
+def shed_tools_conf_option():
+    return planemo_option(
+        "--shed_tool_conf",
+        type=str,
+        help="Location of shed tools conf file for Galaxy.",
+        default=None,
+        use_global_config=True,
+    )
+
+
+def shed_tools_directory_option():
+    return planemo_option(
+        "--shed_tool_path",
+        type=str,
+        help="Location of shed tools directory for Galaxy.",
         default=None,
         use_global_config=True,
     )
@@ -187,6 +216,21 @@ def tool_dependency_dir_option():
     )
 
 
+def job_config_option():
+    return planemo_option(
+        "--job_config_file",
+        type=click.Path(
+            exists=True,
+            file_okay=True,
+            dir_okay=False,
+            resolve_path=True
+        ),
+        help="Job configuration file for Galaxy to target.",
+        default=None,
+        use_global_config=True,
+    )
+
+
 def install_galaxy_option():
     return planemo_option(
         "--install_galaxy",
@@ -649,14 +693,18 @@ def galaxy_config_options():
         test_data_option(),
         tool_data_table_option(),
         dependency_resolvers_option(),
-        tool_dependency_dir_option(),
         brew_dependency_resolution(),
         shed_dependency_resolution(),
         conda_target_options(),
         conda_dependency_resolution(),
         conda_copy_dependencies_option(),
         conda_auto_install_option(),
         conda_auto_init_option(),
+        # Profile options...
+        file_path_option(),
+        database_connection_option(),
+        shed_tools_conf_option(),
+        shed_tools_directory_option(),
     )
 
 
@@ -670,7 +718,9 @@ def galaxy_target_options():
         skip_venv_option(),
         no_cache_galaxy_option(),
         no_cleanup_option(),
+        # Profile options...
         job_config_option(),
+        tool_dependency_dir_option(),
     )
 
 

tests/test_galaxy_config.py

@@ -0,0 +1,46 @@
+"""Unit tests for ``planemo.galaxy.config``."""
+import contextlib
+import os
+
+from .test_utils import TempDirectoryContext, test_context
+
+from planemo.galaxy.config import galaxy_config
+
+
+def test_defaults():
+    """Test by default Galaxy files are stored in temp ``config_directory``."""
+    with _test_galaxy_config() as config:
+        config_directory = config.config_directory
+        _assert_property_is(config, "file_path", os.path.join(config_directory, "files"))
+        conn = "sqlite:///%s/galaxy.sqlite?isolation_level=IMMEDIATE" % config_directory
+        _assert_property_is(config, "database_connection", conn)
+
+
+def test_database_connection_override_path():
+    """Test by default Galaxy files are stored in temp ``config_directory``."""
+    conn = "postgresql://username:password@localhost/mydatabase"
+    with _test_galaxy_config(database_connection=conn) as config:
+        _assert_property_is(config, "database_connection", conn)
+
+
+def test_override_files_path():
+    """Test Galaxy file path overrideable with --file_path."""
+    with TempDirectoryContext() as tdc:
+        with _test_galaxy_config(file_path=tdc.temp_directory) as config:
+            _assert_property_is(config, "file_path", tdc.temp_directory)
+
+
+def _assert_property_is(config, prop, value):
+    env_var = "GALAXY_CONFIG_OVERRIDE_%s" % prop.upper()
+    assert config.env[env_var] == value
+
+
+@contextlib.contextmanager
+def _test_galaxy_config(tool_paths=[], **kwargs):
+    ctx = test_context()
+    with TempDirectoryContext() as tdc:
+        test_data = os.path.join(tdc.temp_directory, "test-data")
+        os.makedirs(test_data)
+        kwargs["test_data"] = test_data
+        with galaxy_config(ctx, tool_paths, **kwargs) as gc:
+            yield gc

tests/test_utils.py

@@ -108,9 +108,7 @@ def _copy_repo(self, name, dest):
 
     @property
     def test_context(self):
-        context = cli.Context()
-        context.planemo_directory = "/tmp/planemo-test-workspace"
-        return context
+        return test_context()
 
 
 class CliShedTestCase(CliTestCase):
@@ -222,6 +220,12 @@ def skip_unless_python_2_7():
     return skip("Python 2.7 required for test.")
 
 
+def test_context():
+    context = cli.Context()
+    context.planemo_directory = "/tmp/planemo-test-workspace"
+    return context
+
+
 __all__ = [
     "TestCase",
     "CliTestCase",