Implement singularity support.

See http://singularity.lbl.gov/ for more information on Singularity and see job_conf.xml.sample_advanced for information on setting up job runners to exploit singularity.

The biggest current caveat is probably that currently these container images need to be setup manually by the admin and the paths hard-coded for each tool in job_conf.xml. There are people who do such manual setups for Docker (https://github.com/phnmnl/container-galaxy-k8s-runtime/blob/develop/config/job_conf.xml) - so it wouldn't be surprising if someone wanted to set this up for singularity as well. That said I'm sure this will be followed up by magic to fetch and convert Docker containers and leverage published singularity containers (such as mulled can now produce (galaxyproject/galaxy-lib#64).

config/job_conf.xml.sample_advanced

@@ -440,6 +440,46 @@
                the deployer. -->
           <!-- <param id="require_container">true</param> -->
         </destination>
+        <destination id="singularity_local" runner="local">
+          <param id="singularity_enabled">true</param>
+          <!-- See the above documentation for docker_volumes, singularity_volumes works
+               the same way.
+          -->
+          <!--
+          <param id="singularity_volumes">$defaults,/mnt/galaxyData/libraries:ro,/mnt/galaxyData/indices:ro</param>
+          -->
+          <!-- You can configure singularity to run using sudo - this probably should not
+               be set and may be removed in the future.
+          -->
+          <!-- <param id="singularity_sudo">false</param> -->
+          <!-- Following option can be used to tweak sudo command used by
+               default. -->
+          <!-- <param id="singularity_sudo_cmd">/usr/bin/sudo -extra_param</param> -->
+          <!-- Pass extra arguments to the singularity exec command not covered by the
+               above options. -->
+          <!-- <param id="singularity_run_extra_arguments"></param> -->
+          <!-- Following command can be used to tweak singularity command. -->
+          <!-- <param id="singularity_cmd">/usr/local/custom_docker/docker</param> -->
+
+          <!-- If deployer wants to use singularity for isolation, but does not
+               trust tool's specified container - a destination wide override
+               can be set. This will cause all jobs on this destination to use
+               that singularity image. -->
+          <!-- <param id="singularity_container_id_override">/path/to/singularity/image</param> -->
+
+          <!-- Likewise, if deployer wants to use singularity for isolation and
+               does trust tool's specified container - but also wants tool's not
+               configured to run in a container the following option can provide
+               a fallback. -->
+          <!-- <param id="singularity_default_container_id">/path/to/singularity/image</param> -->
+
+          <!-- If the destination should be secured to only allow containerized jobs
+               the following parameter may be set for the job destination. Not all,
+               or even most, tools available in Galaxy core or in the Tool Shed
+               support Docker yet so this option may require a lot of extra work for
+               the deployer. -->
+          <!-- <param id="require_container">true</param> -->
+        </destination>
         <destination id="pbs" runner="pbs" tags="mycluster"/>
         <destination id="pbs_longjobs" runner="pbs" tags="mycluster,longjobs">
             <!-- Define parameters that are native to the job runner plugin. -->

lib/galaxy/tools/deps/containers.py

@@ -21,12 +21,14 @@
 from .requirements import ContainerDescription
 from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL
 from ..deps import docker_util
+from ..deps import singularity_util
 
 log = logging.getLogger(__name__)
 
 DOCKER_CONTAINER_TYPE = "docker"
+SINGULARITY_CONTAINER_TYPE = "singularity"
 DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
-ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE]
+ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE, SINGULARITY_CONTAINER_TYPE]
 
 LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = '''
 python << EOF
@@ -315,7 +317,56 @@ def containerize_command(self, command):
         """
 
 
-class DockerContainer(Container):
+class HasDockerLikeVolumes:
+    """Mixin to share functionality related to Docker volume handling.
+
+    Singularity seems to have a fairly compatible syntax for volume handling.
+    """
+
+    def _expand_volume_str(self, value):
+        if not value:
+            return value
+
+        template = string.Template(value)
+        variables = dict()
+
+        def add_var(name, value):
+            if value:
+                variables[name] = os.path.abspath(value)
+
+        add_var("working_directory", self.job_info.working_directory)
+        add_var("job_directory", self.job_info.job_directory)
+        add_var("tool_directory", self.job_info.tool_directory)
+        add_var("galaxy_root", self.app_info.galaxy_root_dir)
+        add_var("default_file_path", self.app_info.default_file_path)
+        add_var("library_import_dir", self.app_info.library_import_dir)
+
+        if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar":
+            # We have a Pulsar job directory, so everything needed (excluding index
+            # files) should be available in job_directory...
+            defaults = "$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw"
+        else:
+            defaults = "$galaxy_root:ro,$tool_directory:ro"
+            if self.job_info.job_directory:
+                defaults += ",$job_directory:ro"
+            if self.app_info.outputs_to_working_directory:
+                # Should need default_file_path (which is a course estimate given
+                # object stores anyway).
+                defaults += ",$working_directory:rw,$default_file_path:ro"
+            else:
+                defaults += ",$working_directory:rw,$default_file_path:rw"
+
+        if self.app_info.library_import_dir:
+            defaults += ",$library_import_dir:ro"
+
+        # Define $defaults that can easily be extended with external library and
+        # index data without deployer worrying about above details.
+        variables["defaults"] = string.Template(defaults).safe_substitute(variables)
+
+        return template.safe_substitute(variables)
+
+
+class DockerContainer(Container, HasDockerLikeVolumes):
 
     def containerize_command(self, command):
         def prop(name, default):
@@ -338,7 +389,7 @@ def prop(name, default):
         if not working_directory:
             raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
 
-        volumes_raw = self.__expand_str(self.destination_info.get("docker_volumes", "$defaults"))
+        volumes_raw = self._expand_volume_str(self.destination_info.get("docker_volumes", "$defaults"))
         # TODO: Remove redundant volumes...
         volumes = docker_util.DockerVolume.volumes_from_str(volumes_raw)
         volumes_from = self.destination_info.get("docker_volumes_from", docker_util.DEFAULT_VOLUMES_FROM)
@@ -394,57 +445,59 @@ def __get_destination_overridable_property(self, name):
         else:
             return getattr(self.app_info, name)
 
-    def __expand_str(self, value):
-        if not value:
-            return value
 
-        template = string.Template(value)
-        variables = dict()
+def docker_cache_path(cache_directory, container_id):
+    file_container_id = container_id.replace("/", "_slash_")
+    cache_file_name = "docker_%s.tar" % file_container_id
+    return os.path.join(cache_directory, cache_file_name)
 
-        def add_var(name, value):
-            if value:
-                variables[name] = os.path.abspath(value)
 
-        add_var("working_directory", self.job_info.working_directory)
-        add_var("job_directory", self.job_info.job_directory)
-        add_var("tool_directory", self.job_info.tool_directory)
-        add_var("galaxy_root", self.app_info.galaxy_root_dir)
-        add_var("default_file_path", self.app_info.default_file_path)
-        add_var("library_import_dir", self.app_info.library_import_dir)
+class SingularityContainer(Container, HasDockerLikeVolumes):
 
-        if self.job_info.job_directory and self.job_info.job_directory_type == "pulsar":
-            # We have a Pulsar job directory, so everything needed (excluding index
-            # files) should be available in job_directory...
-            defaults = "$job_directory:ro,$tool_directory:ro,$job_directory/outputs:rw,$working_directory:rw"
-        else:
-            defaults = "$galaxy_root:ro,$tool_directory:ro"
-            if self.job_info.job_directory:
-                defaults += ",$job_directory:ro"
-            if self.app_info.outputs_to_working_directory:
-                # Should need default_file_path (which is a course estimate given
-                # object stores anyway).
-                defaults += ",$working_directory:rw,$default_file_path:ro"
-            else:
-                defaults += ",$working_directory:rw,$default_file_path:rw"
+    def containerize_command(self, command):
+        def prop(name, default):
+            destination_name = "singularity_%s" % name
+            return self.destination_info.get(destination_name, default)
 
-        if self.app_info.library_import_dir:
-            defaults += ",$library_import_dir:ro"
+        env = []
+        for pass_through_var in self.tool_info.env_pass_through:
+            env.append((pass_through_var, "$%s" % pass_through_var))
 
-        # Define $defaults that can easily be extended with external library and
-        # index data without deployer worrying about above details.
-        variables["defaults"] = string.Template(defaults).safe_substitute(variables)
+        # Allow destinations to explicitly set environment variables just for
+        # docker container. Better approach is to set for destination and then
+        # pass through only what tool needs however. (See todo in ToolInfo.)
+        for key, value in six.iteritems(self.destination_info):
+            if key.startswith("singularity_env_"):
+                real_key = key[len("singularity_env_"):]
+                env.append((real_key, value))
 
-        return template.safe_substitute(variables)
+        working_directory = self.job_info.working_directory
+        if not working_directory:
+            raise Exception("Cannot containerize command [%s] without defined working directory." % working_directory)
 
+        volumes_raw = self._expand_volume_str(self.destination_info.get("singularity_volumes", "$defaults"))
+        volumes = docker_util.DockerVolume.volumes_from_str(volumes_raw)
 
-def docker_cache_path(cache_directory, container_id):
-    file_container_id = container_id.replace("/", "_slash_")
-    cache_file_name = "docker_%s.tar" % file_container_id
-    return os.path.join(cache_directory, cache_file_name)
+        singularity_target_kwds = dict(
+            singularity_cmd=prop("cmd", singularity_util.DEFAULT_SINGULARITY_COMMAND),
+            sudo=asbool(prop("sudo", singularity_util.DEFAULT_SUDO)),
+            sudo_cmd=prop("sudo_cmd", singularity_util.DEFAULT_SUDO_COMMAND),
+        )
+        run_command = singularity_util.build_singularity_run_command(
+            command,
+            self.container_id,
+            volumes=volumes,
+            env=env,
+            working_directory=working_directory,
+            run_extra_arguments=prop("run_extra_arguments", singularity_util.DEFAULT_RUN_EXTRA_ARGUMENTS),
+            **singularity_target_kwds
+        )
+        return run_command
 
 
 CONTAINER_CLASSES = dict(
     docker=DockerContainer,
+    singularity=SingularityContainer,
 )
 
 

lib/galaxy/tools/deps/singularity_util.py

@@ -0,0 +1,58 @@
+from six.moves import shlex_quote
+
+
+DEFAULT_WORKING_DIRECTORY = None
+DEFAULT_SINGULARITY_COMMAND = "singularity"
+DEFAULT_SUDO = False
+DEFAULT_SUDO_COMMAND = "sudo"
+DEFAULT_RUN_EXTRA_ARGUMENTS = None
+
+
+def build_singularity_run_command(
+    container_command,
+    image,
+    volumes=[],
+    env=[],
+    working_directory=DEFAULT_WORKING_DIRECTORY,
+    singularity_cmd=DEFAULT_SINGULARITY_COMMAND,
+    run_extra_arguments=DEFAULT_RUN_EXTRA_ARGUMENTS,
+    sudo=DEFAULT_SUDO,
+    sudo_cmd=DEFAULT_SUDO_COMMAND,
+):
+    command_parts = []
+    # http://singularity.lbl.gov/docs-environment-metadata
+    for (key, value) in env:
+        command_parts.extend(["SINGULARITYENV_%s=%s" % (key, value)])
+    command_parts += _singularity_prefix(
+        singularity_cmd=singularity_cmd,
+        sudo=sudo,
+        sudo_cmd=sudo_cmd,
+    )
+    command_parts.append("exec")
+    for volume in volumes:
+        command_parts.extend(["-B", shlex_quote(str(volume))])
+    if working_directory:
+        command_parts.extend(["--pwd", shlex_quote(working_directory)])
+    if run_extra_arguments:
+        command_parts.append(run_extra_arguments)
+    full_image = image
+    command_parts.append(shlex_quote(full_image))
+    command_parts.append(container_command)
+    return " ".join(command_parts)
+
+
+def _singularity_prefix(
+    singularity_cmd=DEFAULT_SINGULARITY_COMMAND,
+    sudo=DEFAULT_SUDO,
+    sudo_cmd=DEFAULT_SUDO_COMMAND,
+    **kwds
+):
+    """Prefix to issue a singularity command."""
+    command_parts = []
+    if sudo:
+        command_parts.append(sudo_cmd)
+    command_parts.append(singularity_cmd)
+    return command_parts
+
+
+__all__ = ("build_singularity_run_command",)