From 3c5591a2cbe4d3bd7c65f6111b9a943a5a15cc6f Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Tue, 2 Feb 2021 14:17:16 +0100 Subject: [PATCH 01/10] add SARUS changes --- reframe/core/containers.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index b0e2480c59..3a076e9052 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -88,9 +88,6 @@ def validate(self): if self.image is None: raise ContainerError('no image specified') - if not self.commands: - raise ContainerError('no commands specified') - class Docker(ContainerPlatform): '''Container platform backend for running containers with `Docker @@ -119,10 +116,17 @@ class Sarus(ContainerPlatform): #: :default: :class:`False` with_mpi = fields.TypedField(bool) + #: Skip pull of images + #: + #: :type: boolean + #: :default: :class:`False` + skip_pull = fields.TypedField(bool) + def __init__(self): super().__init__() self.with_mpi = False self._command = 'sarus' + self.skip_pull = False def emit_prepare_commands(self): # The format that Sarus uses to call the images is @@ -130,6 +134,9 @@ def emit_prepare_commands(self): # locally from a tar file, the is 'load'. if self.image.startswith('load/'): return [] + # For testing purposes a way to skip the pull of an image. + if self.skip_pull: + return [] return [self._command + ' pull %s' % self.image] @@ -137,14 +144,17 @@ def launch_command(self): super().launch_command() run_opts = ['--mount=type=bind,source="%s",destination="%s"' % mp for mp in self.mount_points] + run_opts.append('--workdir='+self.workdir) if self.with_mpi: run_opts.append('--mpi') run_opts += self.options - run_cmd = self._command + ' run %s %s bash -c ' % (' '.join(run_opts), + run_cmd = self._command + ' run %s %s' % (' '.join(run_opts), self.image) - return run_cmd + "'" + '; '.join( - ['cd ' + self.workdir] + self.commands) + "'" + if self.commands: + return run_cmd + "'" + ' ; '.join(self.commands) + "'" + else: + return run_cmd class Shifter(Sarus): From feb003f024c2e0675c4561beab68eca12cee84c6 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Tue, 2 Feb 2021 14:58:01 +0100 Subject: [PATCH 02/10] rm skip_pull in favor of with_metahub --- reframe/core/containers.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 3a076e9052..135fb2afd0 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -8,7 +8,7 @@ import reframe.core.fields as fields import reframe.utility.typecheck as typ from reframe.core.exceptions import ContainerError - +from tempfile import TemporaryDirectory class ContainerPlatform(abc.ABC): '''The abstract base class of any container platform.''' @@ -116,17 +116,17 @@ class Sarus(ContainerPlatform): #: :default: :class:`False` with_mpi = fields.TypedField(bool) - #: Skip pull of images + #: Using metahub to fetch instance specific images #: #: :type: boolean #: :default: :class:`False` - skip_pull = fields.TypedField(bool) + with_metahub = fields.TypedField(bool) def __init__(self): super().__init__() self.with_mpi = False self._command = 'sarus' - self.skip_pull = False + self.with_metahub = False def emit_prepare_commands(self): # The format that Sarus uses to call the images is @@ -134,11 +134,17 @@ def emit_prepare_commands(self): # locally from a tar file, the is 'load'. if self.image.startswith('load/'): return [] - # For testing purposes a way to skip the pull of an image. - if self.skip_pull: - return [] - - return [self._command + ' pull %s' % self.image] + # Using + if self.with_metahub: + tmpdir = TemporaryDirectory() + return [ + 'docker pull -q mh.qnib.org/'+self.image, + 'docker save -o '+tmpdir+'/image.tar mh.qnib.org/'+self.image, + self._command + ' load ${TMP_DIR}/gromacs.tar mh.qnib.org/'+self.image, + 'rm -rf '+tmpdir + ] + else: + return [self._command + ' pull %s' % self.image] def launch_command(self): super().launch_command() From e17d4ed8dc5d020443638edafd5eabaefc075130 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Tue, 2 Feb 2021 15:00:50 +0100 Subject: [PATCH 03/10] tempfile anyone --- reframe/core/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 135fb2afd0..6e0fdf8c3d 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -8,7 +8,7 @@ import reframe.core.fields as fields import reframe.utility.typecheck as typ from reframe.core.exceptions import ContainerError -from tempfile import TemporaryDirectory +import tempfile class ContainerPlatform(abc.ABC): '''The abstract base class of any container platform.''' @@ -136,7 +136,7 @@ def emit_prepare_commands(self): return [] # Using if self.with_metahub: - tmpdir = TemporaryDirectory() + tmpdir = tempfile.TemporaryDirectory() return [ 'docker pull -q mh.qnib.org/'+self.image, 'docker save -o '+tmpdir+'/image.tar mh.qnib.org/'+self.image, From 346400beacf4977df41b3a20d66adc4ebef10b97 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Tue, 2 Feb 2021 18:10:24 +0100 Subject: [PATCH 04/10] adjust tempdir --- reframe/core/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 6e0fdf8c3d..8e4e6256fe 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -136,7 +136,7 @@ def emit_prepare_commands(self): return [] # Using if self.with_metahub: - tmpdir = tempfile.TemporaryDirectory() + tmpdir = tempfile.mkdtemp() return [ 'docker pull -q mh.qnib.org/'+self.image, 'docker save -o '+tmpdir+'/image.tar mh.qnib.org/'+self.image, From ff868ad58483a32290ac291172488e6bc95896d4 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Tue, 2 Feb 2021 18:56:07 +0100 Subject: [PATCH 05/10] kinda like skip_pull to speed up things --- reframe/core/containers.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 8e4e6256fe..d09f1b4c3d 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -121,18 +121,24 @@ class Sarus(ContainerPlatform): #: :type: boolean #: :default: :class:`False` with_metahub = fields.TypedField(bool) + #: Skipping pull alltogether to speed up runs + #: + #: :type: boolean + #: :default: :class:`False` + skip_pull = fields.TypedField(bool) def __init__(self): super().__init__() self.with_mpi = False self._command = 'sarus' self.with_metahub = False + self.skip_pull = False def emit_prepare_commands(self): # The format that Sarus uses to call the images is # //:. If an image was loaded # locally from a tar file, the is 'load'. - if self.image.startswith('load/'): + if self.image.startswith('load/') or self.skip_pull: return [] # Using if self.with_metahub: From d7bb0254e2e30574c04d22f67958e72362955647 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Wed, 3 Feb 2021 09:42:11 +0100 Subject: [PATCH 06/10] adjust tests --- unittests/test_containers.py | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 5910a50057..8f674c51e6 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -46,17 +46,20 @@ def expected_cmd_mount_points(container_variant): return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "image:tag 'cmd1; cmd2'") elif container_variant == 'Sarus+mpi': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "--mpi image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "--mpi image:tag 'cmd1; cmd2'") elif container_variant == 'Sarus+localimage': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "load/library/image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "load/library/image:tag 'cmd1; cmd2'") elif container_variant == 'Singularity': return ('singularity exec -B"/path/one:/one" -B"/path/two:/two" ' "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") @@ -67,17 +70,20 @@ def expected_cmd_mount_points(container_variant): return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "image:tag bash -c 'cmd1; cmd2'") elif container_variant == 'Shifter+localimage': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "load/library/image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "load/library/image:tag 'cmd1; cmd2'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "--mpi image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--workdir=/stagedir ' + "--mpi image:tag bash -c 'cmd1; cmd2'") @pytest.fixture @@ -98,30 +104,36 @@ def expected_cmd_run_opts(container_variant): elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--workdir=/stagedir ' + "--foo --bar image:tag 'cmd'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--mpi --foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--workdir=/stagedir ' + "--mpi --foo --bar image:tag 'cmd'") elif container_variant == 'Shifter+localimage': return ( 'shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar load/library/image:tag bash -c 'cd /stagedir; cmd'" + '--workdir=/stagedir ' + "--foo --bar load/library/image:tag 'cmd'" ) elif container_variant == 'Sarus': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--workdir=/stagedir ' + "--foo --bar image:tag 'cmd'") elif container_variant == 'Sarus+mpi': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--mpi --foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--workdir=/stagedir ' + "--mpi --foo --bar image:tag 'cmd'") elif container_variant == 'Sarus+localimage': return ( 'sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar load/library/image:tag bash -c 'cd /stagedir; cmd'" + '--workdir=/stagedir ' + "--foo --bar load/library/image:tag 'cmd'" ) elif container_variant == 'Singularity': return ('singularity exec -B"/path/one:/one" ' From 6c71a1565e82184faec7f4bc9748a3af198c4059 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Wed, 3 Feb 2021 09:44:09 +0100 Subject: [PATCH 07/10] add space before cmd --- reframe/core/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index d09f1b4c3d..03ebbdd357 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -164,7 +164,7 @@ def launch_command(self): run_cmd = self._command + ' run %s %s' % (' '.join(run_opts), self.image) if self.commands: - return run_cmd + "'" + ' ; '.join(self.commands) + "'" + return run_cmd + " '" + ' ; '.join(self.commands) + "'" else: return run_cmd From 9a3c0ab4a1c051944f7b01fb4aad6ad425ab70f2 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Wed, 3 Feb 2021 10:17:01 +0100 Subject: [PATCH 08/10] remove empty cmd test for containers --- unittests/test_containers.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 8f674c51e6..c00428b0ae 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -157,13 +157,6 @@ def test_missing_image(container_platform): with pytest.raises(ContainerError): container_platform.validate() - -def test_missing_commands(container_platform): - container_platform.image = 'image:tag' - with pytest.raises(ContainerError): - container_platform.validate() - - def test_prepare_command(container_platform, expected_cmd_prepare): assert container_platform.emit_prepare_commands() == expected_cmd_prepare From 93f19c973b60ea4527487c77194d33af0aabefd2 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Wed, 3 Feb 2021 10:24:12 +0100 Subject: [PATCH 09/10] adjust cmd join in container test --- reframe/core/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 03ebbdd357..d8fc0bf78a 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -164,7 +164,7 @@ def launch_command(self): run_cmd = self._command + ' run %s %s' % (' '.join(run_opts), self.image) if self.commands: - return run_cmd + " '" + ' ; '.join(self.commands) + "'" + return run_cmd + " '" + '; '.join(self.commands) + "'" else: return run_cmd From ac6fd8a6d9b3361cc16fe77da01b14808e393ed6 Mon Sep 17 00:00:00 2001 From: Christian Kniep Date: Wed, 3 Feb 2021 11:47:01 +0100 Subject: [PATCH 10/10] fix unittest --- unittests/test_containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index c00428b0ae..c0cf9528aa 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -71,7 +71,7 @@ def expected_cmd_mount_points(container_variant): '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' '--workdir=/stagedir ' - "image:tag bash -c 'cmd1; cmd2'") + "image:tag 'cmd1; cmd2'") elif container_variant == 'Shifter+localimage': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' @@ -83,7 +83,7 @@ def expected_cmd_mount_points(container_variant): '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' '--workdir=/stagedir ' - "--mpi image:tag bash -c 'cmd1; cmd2'") + "--mpi image:tag 'cmd1; cmd2'") @pytest.fixture