From 75e2f4f2b5e6e49b450f6477ab5985108489298c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 14:00:08 +0100 Subject: [PATCH 01/10] tripple the size for small containers --- galaxy/tools/deps/mulled/invfile.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/tools/deps/mulled/invfile.lua b/galaxy/tools/deps/mulled/invfile.lua index 4370e2c..e0dcf73 100644 --- a/galaxy/tools/deps/mulled/invfile.lua +++ b/galaxy/tools/deps/mulled/invfile.lua @@ -104,8 +104,8 @@ if VAR.SINGULARITY ~= '' then .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true}) .withConfig({entrypoint = {'/bin/sh', '-c'}}) -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size - .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*2)); else echo $(($size+$size*7/10)); fi") - .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*2)); else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) + .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*3)); else echo $(($size+$size*7/10)); fi") + .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*3)); else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity') .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME) end From 60a4d584433ce59537812bfedb5f36ad8da034d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 14:18:47 +0100 Subject: [PATCH 02/10] use miniconda3 woth python3 as default --- galaxy/tools/deps/mulled/invfile.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/tools/deps/mulled/invfile.lua b/galaxy/tools/deps/mulled/invfile.lua index e0dcf73..99102fa 100644 --- a/galaxy/tools/deps/mulled/invfile.lua +++ b/galaxy/tools/deps/mulled/invfile.lua @@ -42,7 +42,7 @@ end local conda_image = VAR.CONDA_IMAGE if conda_image == '' then - conda_image = 'continuumio/miniconda:latest' + conda_image = 'continuumio/miniconda3:latest' end From 09f0417c8e18530395b58a6d4047ff5e679f3d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 15:06:24 +0100 Subject: [PATCH 03/10] fail early --- galaxy/tools/deps/mulled/mulled_build_files.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/galaxy/tools/deps/mulled/mulled_build_files.py b/galaxy/tools/deps/mulled/mulled_build_files.py index 3ae94e6..4145c87 100644 --- a/galaxy/tools/deps/mulled/mulled_build_files.py +++ b/galaxy/tools/deps/mulled/mulled_build_files.py @@ -36,7 +36,7 @@ def main(argv=None): args = parser.parse_args() for (targets, image_build, name_override) in generate_targets(args.files): try: - mull_targets( + ret = mull_targets( targets, image_build=image_build, name_override=name_override, @@ -44,6 +44,9 @@ def main(argv=None): ) except BuildExistsException: continue + if ret > 0: + sys.exit(ret) + def generate_targets(target_source): From 31cf7e42746ff23efc89293a30dce3453bfe343f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 15:15:08 +0100 Subject: [PATCH 04/10] import sys --- galaxy/tools/deps/mulled/mulled_build_files.py | 1 + 1 file changed, 1 insertion(+) diff --git a/galaxy/tools/deps/mulled/mulled_build_files.py b/galaxy/tools/deps/mulled/mulled_build_files.py index 4145c87..d5211a1 100644 --- a/galaxy/tools/deps/mulled/mulled_build_files.py +++ b/galaxy/tools/deps/mulled/mulled_build_files.py @@ -15,6 +15,7 @@ import collections import glob import os +import sys from ._cli import arg_parser from .mulled_build import ( From ca9d70cb565fb3d84c744de6a5e35bfcf79c42b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 16:45:08 +0100 Subject: [PATCH 05/10] ok, smallest container is 20 --- galaxy/tools/deps/mulled/invfile.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/tools/deps/mulled/invfile.lua b/galaxy/tools/deps/mulled/invfile.lua index 99102fa..b877c34 100644 --- a/galaxy/tools/deps/mulled/invfile.lua +++ b/galaxy/tools/deps/mulled/invfile.lua @@ -104,8 +104,8 @@ if VAR.SINGULARITY ~= '' then .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true}) .withConfig({entrypoint = {'/bin/sh', '-c'}}) -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size - .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*3)); else echo $(($size+$size*7/10)); fi") - .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo $(($size*3)); else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) + .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo 20; else echo $(($size+$size*7/10)); fi") + .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity') .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME) end From d00001690a0225c42065cc2c0f2184fba909b47a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 16:52:14 +0100 Subject: [PATCH 06/10] add PrintProgress class to trick travis for long running downloads --- galaxy/tools/deps/mulled/util.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/galaxy/tools/deps/mulled/util.py b/galaxy/tools/deps/mulled/util.py index 46ede34..ee1d3a2 100644 --- a/galaxy/tools/deps/mulled/util.py +++ b/galaxy/tools/deps/mulled/util.py @@ -207,6 +207,27 @@ def v2_image_name(targets, image_build=None, name_override=None): return "mulled-v2-%s%s" % (package_hash.hexdigest(), suffix) +class PrintProgress: + def __init__(self): + self.thread = threading.Thread(target=self.progress) + self.stop = False + + def progress(self): + while not self.stop: + print(".", end="") + sys.stdout.flush() + time.sleep(60) + print("") + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.stop = True + self.thread.join() + + image_name = v1_image_name # deprecated __all__ = ( From 7b9b5b8c3354a6c9110344411a4bab6b445371cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 16:55:12 +0100 Subject: [PATCH 07/10] use PrintProgress() --- galaxy/tools/deps/mulled/mulled_build.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/galaxy/tools/deps/mulled/mulled_build.py b/galaxy/tools/deps/mulled/mulled_build.py index 5d362f1..2bbb664 100644 --- a/galaxy/tools/deps/mulled/mulled_build.py +++ b/galaxy/tools/deps/mulled/mulled_build.py @@ -33,6 +33,7 @@ conda_build_target_str, create_repository, quay_repository, + PrintProgress, v1_image_name, v2_image_name, ) @@ -242,7 +243,8 @@ def mull_targets( with open(os.path.join(singularity_image_dir, 'Singularity'), 'w+') as sin_def: fill_template = SINGULARITY_TEMPLATE % {'container_test': test} sin_def.write(fill_template) - ret = involucro_context.exec_command(involucro_args) + with PrintProgress(): + ret = involucro_context.exec_command(involucro_args) if singularity: # we can not remove this folder as it contains the image wich is owned by root pass From eefaf60eea0148384297f9052f6342eaf319dc66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 17:00:43 +0100 Subject: [PATCH 08/10] smaller then 10 --- galaxy/tools/deps/mulled/invfile.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/galaxy/tools/deps/mulled/invfile.lua b/galaxy/tools/deps/mulled/invfile.lua index b877c34..4b771e4 100644 --- a/galaxy/tools/deps/mulled/invfile.lua +++ b/galaxy/tools/deps/mulled/invfile.lua @@ -104,8 +104,8 @@ if VAR.SINGULARITY ~= '' then .withHostConfig({binds = {"build:/data",singularity_image_dir .. ":/import"}, privileged = true}) .withConfig({entrypoint = {'/bin/sh', '-c'}}) -- for small containers (less than 7MB), double the size otherwise, add a little bit more as half the conda size - .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo 20; else echo $(($size+$size*7/10)); fi") - .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '7' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) + .run("size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi") + .run("singularity create --size `size=$(du -sc /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)}' ) && if [ $size -lt '10' ]; then echo 20; else echo $(($size+$size*7/10)); fi` /import/" .. VAR.SINGULARITY_IMAGE_NAME) .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity') .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME) end From c12ad2906d25c8011c74154a8b1328b6c6ff724f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 17:05:04 +0100 Subject: [PATCH 09/10] missing imports --- galaxy/tools/deps/mulled/util.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/galaxy/tools/deps/mulled/util.py b/galaxy/tools/deps/mulled/util.py index ee1d3a2..d000d53 100644 --- a/galaxy/tools/deps/mulled/util.py +++ b/galaxy/tools/deps/mulled/util.py @@ -3,6 +3,9 @@ import collections import hashlib +import time +import threading +import sys from distutils.version import LooseVersion From e067e01fb7ccf96c07fd01541aa6ed20c8bce8fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Wed, 22 Nov 2017 17:59:52 +0100 Subject: [PATCH 10/10] remove r-channel --- galaxy/tools/deps/mulled/mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy/tools/deps/mulled/mulled_build.py b/galaxy/tools/deps/mulled/mulled_build.py index 2bbb664..b2985a3 100644 --- a/galaxy/tools/deps/mulled/mulled_build.py +++ b/galaxy/tools/deps/mulled/mulled_build.py @@ -41,7 +41,7 @@ DIRNAME = os.path.dirname(__file__) DEFAULT_CHANNEL = "bioconda" -DEFAULT_EXTRA_CHANNELS = ["conda-forge", "r"] +DEFAULT_EXTRA_CHANNELS = ["conda-forge"] DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}" DEFAULT_BINDS = ["build/dist:/usr/local/"]