Merge pull request #4207 from jmchilton/mulled_container_fixes

Various mulled container fixes.

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -31,13 +31,27 @@
 
 CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
 CachedV1MulledImageMultiTarget = collections.namedtuple("CachedV1MulledImageMultiTarget", ["hash", "build", "image_identifier"])
-CachedV2MulledImageMultiTarget = collections.namedtuple("CachedV2MulledImageMultiTarget", ["package_hash", "version_hash", "build", "image_identifier"])
+CachedV2MulledImageMultiTarget = collections.namedtuple("CachedV2MulledImageMultiTarget", ["image_name", "version_hash", "build", "image_identifier"])
 
 CachedMulledImageSingleTarget.multi_target = False
 CachedV1MulledImageMultiTarget.multi_target = "v1"
 CachedV2MulledImageMultiTarget.multi_target = "v2"
 
 
+@property
+def _package_hash(target):
+    # Make this work for Singularity file name or fully qualified Docker repository
+    # image names.
+    image_name = target.image_name
+    if "/" not in image_name:
+        return image_name
+    else:
+        return image_name.rsplit("/")[-1]
+
+
+CachedV2MulledImageMultiTarget.package_hash = _package_hash
+
+
 def list_docker_cached_mulled_images(namespace=None, hash_func="v2"):
     command = build_docker_images_command(truncate=True, sudo=False)
     command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
@@ -47,15 +61,15 @@ def list_docker_cached_mulled_images(namespace=None, hash_func="v2"):
     def output_line_to_image(line):
         image_name, version = line.split(" ", 1)
         identifier = "%s:%s" % (image_name, version)
-        image = identifier_to_cached_target(identifier, hash_func)
+        image = identifier_to_cached_target(identifier, hash_func, namespace=namespace)
         return image
 
     # TODO: Sort on build ...
     raw_images = [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
     return [i for i in raw_images if i is not None]
 
 
-def identifier_to_cached_target(identifier, hash_func):
+def identifier_to_cached_target(identifier, hash_func, namespace=None):
     if ":" in identifier:
         image_name, version = identifier.rsplit(":", 1)
     else:
@@ -66,7 +80,10 @@ def identifier_to_cached_target(identifier, hash_func):
         version = None
 
     image = None
-    if image_name.startswith("mulled-v1-"):
+    prefix = ""
+    if namespace is not None:
+        prefix = "quay.io/%s/" % namespace
+    if image_name.startswith(prefix + "mulled-v1-"):
         if hash_func == "v2":
             return None
 
@@ -75,7 +92,7 @@ def identifier_to_cached_target(identifier, hash_func):
         if version and version.isdigit():
             build = version
         image = CachedV1MulledImageMultiTarget(hash, build, identifier)
-    elif image_name.startswith("mulled-v2-"):
+    elif image_name.startswith(prefix + "mulled-v2-"):
         if hash_func == "v1":
             return None
 
@@ -202,7 +219,7 @@ class CachedMulledDockerContainerResolver(ContainerResolver):
     resolver_type = "cached_mulled"
     container_type = "docker"
 
-    def __init__(self, app_info=None, namespace=None, hash_func="v2"):
+    def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
         super(CachedMulledDockerContainerResolver, self).__init__(app_info)
         self.namespace = namespace
         self.hash_func = hash_func
@@ -293,7 +310,14 @@ def tags_if_available(image_name):
                 base_image_name = v2_image_name(targets)
                 tags = tags_if_available(base_image_name)
                 if tags:
-                    name = "%s:%s" % (base_image_name, tags[0])
+                    if ":" in base_image_name:
+                        # base_image_name of form <package_hash>:<version_hash>, expand tag
+                        # to include build number in tag.
+                        name = "%s:%s" % (base_image_name.split(":")[0], tags[0])
+                    else:
+                        # base_image_name of form <package_hash>, simply add build number
+                        # as tag to fully qualify image.
+                        name = "%s:%s" % (base_image_name, tags[0])
             elif self.hash_func == "v1":
                 base_image_name = v1_image_name(targets)
                 tags = tags_if_available(base_image_name)

lib/galaxy/tools/deps/containers.py

@@ -221,7 +221,7 @@ def __default_containers_resolvers(self):
         ]
         if self.enable_beta_mulled_containers:
             default_resolvers.extend([
-                CachedMulledDockerContainerResolver(self.app_info),
+                CachedMulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
                 MulledDockerContainerResolver(self.app_info, namespace="biocontainers"),
                 BuildMulledDockerContainerResolver(self.app_info),
                 CachedMulledSingularityContainerResolver(self.app_info),

lib/galaxy/tools/deps/mulled/invfile.lua

@@ -103,10 +103,9 @@ if VAR.SINGULARITY ~= '' then
         .using(singularity_image)
         .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true})
         .withConfig({entrypoint = {'/bin/sh', '-c'}})
-        -- this will create a container that is the size of our conda dependencies + 20MiB
-        -- The 20 MiB can be improved at some point, but this seems to work for now.
-        .run("du -sc  /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)+20}'")
-        .run("singularity create --size `du -sc  /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)+20}'` /import/" .. VAR.SINGULARITY_IMAGE_NAME)
+        -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size
+        .run("size=$(du -sc  /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*2)); else  echo $(($size+$size*7/10)); fi")
+        .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*2)); else  echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME)
         .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity')
         .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME)
 end

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -393,7 +393,7 @@ def args_to_mull_targets_kwds(args):
         kwds["rebuild"] = args.rebuild
     if hasattr(args, "hash"):
         kwds["hash_func"] = args.hash
-    if hasattr(args, "singularity_image_dir"):
+    if hasattr(args, "singularity_image_dir") and args.singularity_image_dir:
         kwds["singularity_image_dir"] = args.singularity_image_dir
 
     kwds["involucro_context"] = context_from_args(args)