From 08e57f495ebbfa254783777bae75590af8165f53 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 15 Feb 2021 17:18:27 +0100 Subject: [PATCH 01/16] Improve the implementation of Container Platforms * Allow for custom pulling commands for container images. * Respect the commands defined inside the container image. --- reframe/core/containers.py | 127 ++++++++++++++++++++-------- unittests/test_containers.py | 159 +++++++++++++++++++++-------------- 2 files changed, 187 insertions(+), 99 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index b0e2480c59..53f4c15dad 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -19,11 +19,31 @@ class ContainerPlatform(abc.ABC): #: :default: :class:`None` image = fields.TypedField(str, type(None)) - #: The commands to be executed within the container. + #: The command to be executed within the container. #: - #: :type: :class:`list[str]` - #: :default: ``[]`` - commands = fields.TypedField(typ.List[str]) + #: If no command is given, then the default command of the corresponding + #: container image is going to be executed. + #: + #: ..versionchanged:: 3.4.2 + #: Changed the attribute name from `commands` to `command` and its type + #: to a string. + #: + #: :type: :class:`str` or :class:`None` + #: :default: :class:`None` + command = fields.TypedField(str, type(None)) + + #: The pull command to be used to pull the container image. + #: + #: If an empty string is given as the pull command, then the default + #: pull command of the corresponding container platform is going to be + #: used to pull the image. If set to :class:`None`, then no pull action + #: is going to be performed by the container platform. + #: + #: ..versionadded:: 3.4.2 + #: + #: :type: :class:`str` or :class:`None` + #: :default: ``''`` + pull_command = fields.TypedField(str, type(None)) #: List of mount point pairs for directories to mount inside the container. #: @@ -44,7 +64,8 @@ class ContainerPlatform(abc.ABC): #: #: This is the directory where the test's stage directory is mounted inside #: the container. This directory is always mounted regardless if - #: :attr:`mount_points` is set or not. + #: :attr:`mount_points` is set or not. If set to :class:`None` then the + #: the default working directory of the given container image is used. #: #: :type: :class:`str` #: :default: ``/rfm_workdir`` @@ -52,9 +73,10 @@ class ContainerPlatform(abc.ABC): def __init__(self): self.image = None - self.commands = [] + self.command = None self.mount_points = [] self.options = [] + self.pull_command = '' self.workdir = '/rfm_workdir' @abc.abstractmethod @@ -88,25 +110,28 @@ def validate(self): if self.image is None: raise ContainerError('no image specified') - if not self.commands: - raise ContainerError('no commands specified') - class Docker(ContainerPlatform): '''Container platform backend for running containers with `Docker `__.''' def emit_prepare_commands(self): + if self.pull_command == '': + return [f'docker pull {self.image}'] + elif self.pull_command: + return [self.pull_command] + return [] def launch_command(self): super().launch_command() - run_opts = ['-v "%s":"%s"' % mp for mp in self.mount_points] + run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] + run_opts += ['-v "${PWD}":"/rfm_workdir"'] run_opts += self.options - run_cmd = 'docker run --rm %s %s bash -c ' % (' '.join(run_opts), - self.image) - return run_cmd + "'" + '; '.join( - ['cd ' + self.workdir] + self.commands) + "'" + workdir_opt = f'--workdir="{self.workdir}" ' if self.workdir else '' + + return (f'docker run --rm {workdir_opt}{" ".join(run_opts)} ' + f'{self.image} {self.command or ""}').rstrip() class Sarus(ContainerPlatform): @@ -122,39 +147,67 @@ class Sarus(ContainerPlatform): def __init__(self): super().__init__() self.with_mpi = False - self._command = 'sarus' def emit_prepare_commands(self): - # The format that Sarus uses to call the images is - # //:. If an image was loaded - # locally from a tar file, the is 'load'. - if self.image.startswith('load/'): - return [] + if self.pull_command == '': + return [f'sarus pull {self.image}'] + elif self.pull_command: + return [self.pull_command] - return [self._command + ' pull %s' % self.image] + return [] def launch_command(self): super().launch_command() - run_opts = ['--mount=type=bind,source="%s",destination="%s"' % - mp for mp in self.mount_points] + run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' + for mp in self.mount_points] + run_opts += ['--mount=type=bind,source="${PWD}",' + 'destination="/rfm_workdir"'] if self.with_mpi: run_opts.append('--mpi') run_opts += self.options - run_cmd = self._command + ' run %s %s bash -c ' % (' '.join(run_opts), - self.image) - return run_cmd + "'" + '; '.join( - ['cd ' + self.workdir] + self.commands) + "'" + + workdir_opt = f'--workdir="{self.workdir}" ' if self.workdir else '' + return (f'sarus run {workdir_opt}{" ".join(run_opts)} {self.image} ' + f'{self.command or ""}').rstrip() -class Shifter(Sarus): +class Shifter(ContainerPlatform): '''Container platform backend for running containers with `Shifter `__. ''' + #: Enable MPI support when launching the container. + #: + #: :type: boolean + #: :default: :class:`False` + with_mpi = fields.TypedField(bool) + def __init__(self): super().__init__() - self._command = 'shifter' + self.with_mpi = False + + def emit_prepare_commands(self): + if self.pull_command == '': + return [f'shifter pull {self.image}'] + elif self.pull_command: + return [self.pull_command] + + return [] + + def launch_command(self): + super().launch_command() + run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' + for mp in self.mount_points] + run_opts += ['--mount=type=bind,source="${PWD}",' + 'destination="/rfm_workdir"'] + if self.with_mpi: + run_opts.append('--mpi') + + run_opts += self.options + + return (f"shifter run {' '.join(run_opts)} {self.image} bash -c '" + f"cd {self.workdir};{self.command or ''}'") class Singularity(ContainerPlatform): @@ -176,15 +229,19 @@ def emit_prepare_commands(self): def launch_command(self): super().launch_command() - run_opts = ['-B"%s:%s"' % mp for mp in self.mount_points] + run_opts = [f'-B"{mp[0]}:{mp[1]}"' for mp in self.mount_points] + run_opts += ['-B"${PWD}:/rfm_workdir"'] if self.with_cuda: run_opts.append('--nv') run_opts += self.options - run_cmd = 'singularity exec %s %s bash -c ' % (' '.join(run_opts), - self.image) - return run_cmd + "'" + '; '.join( - ['cd ' + self.workdir] + self.commands) + "'" + workdir_cmd = f'--workdir="{self.workdir}" ' if self.workdir else '' + if self.command: + return (f'singularity exec {workdir_cmd}{" ".join(run_opts)} ' + f'{self.image} {self.command}') + + return (f'singularity run {workdir_cmd}{" ".join(run_opts)} ' + f'{self.image}') class ContainerPlatformField(fields.TypedField): @@ -197,6 +254,6 @@ def __set__(self, obj, value): value = globals()[value]() except KeyError: raise ValueError( - 'unknown container platform: %s' % value) from None + f'unknown container platform: {value}') from None super().__set__(obj, value) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 5910a50057..af38937575 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -10,10 +10,9 @@ @pytest.fixture(params=[ - 'Docker', - 'Sarus', 'Sarus+mpi', 'Sarus+localimage', - 'Shifter', 'Shifter+mpi', 'Shifter+localimage', - 'Singularity', 'Singularity+cuda' + 'Docker', 'Docker+nocommand', 'Docker+nopull', 'Sarus', 'Sarus+nocommand', + 'Sarus+nopull', 'Sarus+custompull', 'Sarus+mpi', 'Shifter', 'Shifter+mpi', + 'Singularity', 'Singularity+cuda', 'Singularity+nocommand' ]) def container_variant(request): return request.param @@ -23,131 +22,164 @@ def container_variant(request): def container_platform(container_variant): name = container_variant.split('+')[0] ret = containers.__dict__[name]() + if '+nocommand' not in container_variant: + ret.command = 'cmd' + if '+mpi' in container_variant: ret.with_mpi = True if '+cuda' in container_variant: ret.with_cuda = True - if '+localimage' in container_variant: + if container_variant == 'Sarus+custompull': ret.image = 'load/library/image:tag' else: ret.image = 'image:tag' + if '+nopull' in container_variant: + ret.pull_command = None + + if container_variant == 'Sarus+custompull': + ret.pull_command = ('docker pull registry/image:tag; ' + 'docker save -o local_image.tar; ' + 'sarus load local_image.tar image:tag') + return ret @pytest.fixture def expected_cmd_mount_points(container_variant): - if container_variant == 'Docker': - return ('docker run --rm -v "/path/one":"/one" -v "/path/two":"/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") - elif container_variant == 'Sarus': - return ('sarus run ' + if container_variant in {'Docker', 'Docker+nopull'}: + return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + '-v "/path/two":"/two" -v "${PWD}":"/rfm_workdir" ' + 'image:tag cmd') + elif container_variant == 'Docker+nocommand': + return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + '-v "/path/two":"/two" -v "${PWD}":"/rfm_workdir" image:tag') + elif container_variant in {'Sarus', 'Sarus+nopull'}: + return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") - elif container_variant == 'Sarus+mpi': - return ('sarus run ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + 'image:tag cmd') + elif container_variant == 'Sarus+custompull': + return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "--mpi image:tag bash -c 'cd /stagedir; cmd1; cmd2'") - elif container_variant == 'Sarus+localimage': - return ('sarus run ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + 'load/library/image:tag cmd') + elif container_variant == 'Sarus+nocommand': + return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "load/library/image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + 'image:tag') + elif container_variant == 'Sarus+mpi': + return ('sarus run --workdir="/stagedir" ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + '--mpi image:tag cmd') elif container_variant == 'Singularity': - return ('singularity exec -B"/path/one:/one" -B"/path/two:/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" image:tag cmd') elif container_variant == 'Singularity+cuda': - return ('singularity exec -B"/path/one:/one" -B"/path/two:/two" ' - "--nv image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" ' + '--nv image:tag cmd') + elif container_variant == 'Singularity+nocommand': + return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' + '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "image:tag bash -c 'cd /stagedir; cmd1; cmd2'") - elif container_variant == 'Shifter+localimage': - return ('shifter run ' - '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="/path/two",destination="/two" ' - "load/library/image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + "image:tag bash -c 'cd /stagedir;cmd'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "--mpi image:tag bash -c 'cd /stagedir; cmd1; cmd2'") + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + "--mpi image:tag bash -c 'cd /stagedir;cmd'") @pytest.fixture def expected_cmd_prepare(container_variant): + if container_variant in ('Docker', 'Docker+nocommand'): + return ['docker pull image:tag'] if container_variant in ('Shifter', 'Shifter+mpi'): return ['shifter pull image:tag'] - elif container_variant in ('Sarus', 'Sarus+mpi'): + elif container_variant in ('Sarus', 'Sarus+nocommand', 'Sarus+mpi'): return ['sarus pull image:tag'] + elif container_variant == 'Sarus+custompull': + return [ + 'docker pull registry/image:tag; ' + 'docker save -o local_image.tar; ' + 'sarus load local_image.tar image:tag' + ] else: return [] @pytest.fixture def expected_cmd_run_opts(container_variant): - if container_variant == 'Docker': - return ('docker run --rm -v "/path/one":"/one" --foo --bar ' - "image:tag bash -c 'cd /stagedir; cmd'") + if container_variant in {'Docker', 'Docker+nopull'}: + return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + '-v "${PWD}":"/rfm_workdir" --foo --bar image:tag cmd') + if container_variant == 'Docker+nocommand': + return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + '-v "${PWD}":"/rfm_workdir" --foo --bar image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + "--foo --bar image:tag bash -c 'cd /stagedir;cmd'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--mpi --foo --bar image:tag bash -c 'cd /stagedir; cmd'") - elif container_variant == 'Shifter+localimage': - return ( - 'shifter run ' - '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar load/library/image:tag bash -c 'cd /stagedir; cmd'" - ) - elif container_variant == 'Sarus': - return ('sarus run ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + "--mpi --foo --bar image:tag bash -c 'cd /stagedir;cmd'") + elif container_variant in {'Sarus', 'Sarus+nopull'}: + return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + '--foo --bar image:tag cmd') + elif container_variant == 'Sarus+nocommand': + return ('sarus run --workdir="/stagedir" ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + '--foo --bar image:tag') elif container_variant == 'Sarus+mpi': - return ('sarus run ' + return ('sarus run --workdir="/stagedir" ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + '--mpi --foo --bar image:tag cmd') + elif container_variant == 'Sarus+custompull': + return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--mpi --foo --bar image:tag bash -c 'cd /stagedir; cmd'") - elif container_variant == 'Sarus+localimage': - return ( - 'sarus run ' - '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar load/library/image:tag bash -c 'cd /stagedir; cmd'" - ) + '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' + '--foo --bar load/library/image:tag cmd') elif container_variant == 'Singularity': - return ('singularity exec -B"/path/one:/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir; cmd'") + return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + '-B"${PWD}:/rfm_workdir" --foo --bar image:tag cmd') elif container_variant == 'Singularity+cuda': - return ('singularity exec -B"/path/one:/one" ' - "--nv --foo --bar image:tag bash -c 'cd /stagedir; cmd'") + return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + '-B"${PWD}:/rfm_workdir" --nv --foo --bar image:tag cmd') + elif container_variant == 'Singularity+nocommand': + return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' + '-B"${PWD}:/rfm_workdir" --foo --bar image:tag') def test_mount_points(container_platform, expected_cmd_mount_points): container_platform.mount_points = [('/path/one', '/one'), ('/path/two', '/two')] - container_platform.commands = ['cmd1', 'cmd2'] container_platform.workdir = '/stagedir' assert container_platform.launch_command() == expected_cmd_mount_points def test_missing_image(container_platform): container_platform.image = None - container_platform.commands = ['cmd'] - with pytest.raises(ContainerError): - container_platform.validate() - - -def test_missing_commands(container_platform): - container_platform.image = 'image:tag' with pytest.raises(ContainerError): container_platform.validate() @@ -157,7 +189,6 @@ def test_prepare_command(container_platform, expected_cmd_prepare): def test_run_opts(container_platform, expected_cmd_run_opts): - container_platform.commands = ['cmd'] container_platform.mount_points = [('/path/one', '/one')] container_platform.workdir = '/stagedir' container_platform.options = ['--foo', '--bar'] From 51d73d1db2ab1270c92d051868591a2556e990a3 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 16 Feb 2021 16:35:16 +0100 Subject: [PATCH 02/16] Respect working directory of container --- docs/tutorial_advanced.rst | 22 +++++++++---- reframe/core/containers.py | 30 ++++++++--------- reframe/core/pipeline.py | 2 +- .../advanced/containers/container_test.py | 7 ++-- unittests/test_containers.py | 33 ++++++------------- 5 files changed, 43 insertions(+), 51 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 775864447c..bafdef0411 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -686,27 +686,28 @@ The following test will use a Singularity container to run: .. literalinclude:: ../tutorials/advanced/containers/container_test.py :lines: 6- - :emphasize-lines: 11-16 + :emphasize-lines: 11-15 A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform ` attribute. This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. In this case, the test will be using `Singularity `__ as a container platform. If such a platform is not `configured `__ for the current system, the test will fail. -As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container by setting the :attr:`image ` and the :attr:`commands ` container platform attributes. -These two attributes are mandatory for container-based checks. +As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image `. +The default command that the container runs can be overwritten by setting the :attr:`command ` is the only mandatory attribute for container-based checks. It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. ReFrame will run the container as follows: .. code-block:: console - singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' + singularity exec -B"/path/to/test/stagedir:/rfm_stagedir" --workdir="/rfm_stagedir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' -By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. -The user commands are then run from that directory one after the other. +By default ReFrame will mount the stage directory of the test under ``/rfm_stagedir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. -Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: +Users may also change the default working directory of the container using the :attr:`workdir ` attribute: Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: .. code-block:: python @@ -714,5 +715,12 @@ Besides the stage directory, additional mount points can be specified through th self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), ('/path/to/host/dir2', '/path/to/container/mount_point2')] +.. tip:: + + The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_stagedir`` inside the container where the user can copy directories/files as needed. + The aforementioned directories/files will then be available inside the stage directory after the container execution finishes. + This is very useful if the above directories/files are going to be used for the sanity/performance checks. + If this is the case, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. + For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`. diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 53f4c15dad..14d4c964cf 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -13,6 +13,10 @@ class ContainerPlatform(abc.ABC): '''The abstract base class of any container platform.''' + #: The default mount location of the test case stage directory inside the + #: container + RFM_STAGEDIR = '/rfm_stagedir' + #: The container image to be used for running the test. #: #: :type: :class:`str` or :class:`None` @@ -48,7 +52,9 @@ class ContainerPlatform(abc.ABC): #: List of mount point pairs for directories to mount inside the container. #: #: Each mount point is specified as a tuple of - #: ``(/path/in/host, /path/in/container)``. + #: ``(/path/in/host, /path/in/container)``. The stage directory of the + #: ReFrame test is always mounted under ``/rfm_stagedir`` inside the + #: container, independelty of this field. #: #: :type: :class:`list[tuple[str, str]]` #: :default: ``[]`` @@ -60,15 +66,13 @@ class ContainerPlatform(abc.ABC): #: :default: ``[]`` options = fields.TypedField(typ.List[str]) - #: The working directory of ReFrame inside the container. + #: The working directory inside the container. #: - #: This is the directory where the test's stage directory is mounted inside - #: the container. This directory is always mounted regardless if - #: :attr:`mount_points` is set or not. If set to :class:`None` then the - #: the default working directory of the given container image is used. + #: If set to :class:`None` then the default working directory of the given + #: container image is used. #: #: :type: :class:`str` - #: :default: ``/rfm_workdir`` + #: :default: :class:None workdir = fields.TypedField(str, type(None)) def __init__(self): @@ -77,7 +81,7 @@ def __init__(self): self.mount_points = [] self.options = [] self.pull_command = '' - self.workdir = '/rfm_workdir' + self.workdir = None @abc.abstractmethod def emit_prepare_commands(self): @@ -126,7 +130,6 @@ def emit_prepare_commands(self): def launch_command(self): super().launch_command() run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] - run_opts += ['-v "${PWD}":"/rfm_workdir"'] run_opts += self.options workdir_opt = f'--workdir="{self.workdir}" ' if self.workdir else '' @@ -160,8 +163,6 @@ def launch_command(self): super().launch_command() run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' for mp in self.mount_points] - run_opts += ['--mount=type=bind,source="${PWD}",' - 'destination="/rfm_workdir"'] if self.with_mpi: run_opts.append('--mpi') @@ -199,15 +200,13 @@ def launch_command(self): super().launch_command() run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' for mp in self.mount_points] - run_opts += ['--mount=type=bind,source="${PWD}",' - 'destination="/rfm_workdir"'] if self.with_mpi: run_opts.append('--mpi') run_opts += self.options - + workdir_opt = f'cd {self.workdir};' if self.workdir else '' return (f"shifter run {' '.join(run_opts)} {self.image} bash -c '" - f"cd {self.workdir};{self.command or ''}'") + f"{workdir_opt}{self.command or ''}'") class Singularity(ContainerPlatform): @@ -230,7 +229,6 @@ def emit_prepare_commands(self): def launch_command(self): super().launch_command() run_opts = [f'-B"{mp[0]}:{mp[1]}"' for mp in self.mount_points] - run_opts += ['-B"${PWD}:/rfm_workdir"'] if self.with_cuda: run_opts.append('--nv') diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ef9b66e477..65370a36c2 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -1325,7 +1325,7 @@ def run(self): self.container_platform.validate() self.container_platform.mount_points += [ - (self._stagedir, self.container_platform.workdir) + (self._stagedir, self.container_platform.RFM_STAGEDIR) ] # We replace executable and executable_opts in case of containers diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index b2f469c36c..a05224a30c 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -15,10 +15,9 @@ def __init__(self): self.valid_prog_environs = ['builtin'] self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.commands = [ - 'pwd', 'ls', 'cat /etc/os-release' - ] - self.container_platform.workdir = '/workdir' + self.container_platform.command = ("bash -c 'pwd; ls;" + "cat /etc/os-release'") + self.container_platform.workdir = '/rfm_stagedir' self.sanity_patterns = sn.all([ sn.assert_found(r'^' + self.container_platform.workdir, self.stdout), diff --git a/unittests/test_containers.py b/unittests/test_containers.py index af38937575..29ccc74c7b 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -51,56 +51,49 @@ def container_platform(container_variant): def expected_cmd_mount_points(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' - '-v "/path/two":"/two" -v "${PWD}":"/rfm_workdir" ' - 'image:tag cmd') + '-v "/path/two":"/two" image:tag cmd') elif container_variant == 'Docker+nocommand': return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' - '-v "/path/two":"/two" -v "${PWD}":"/rfm_workdir" image:tag') + '-v "/path/two":"/two" image:tag') elif container_variant in {'Sarus', 'Sarus+nopull'}: return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' 'image:tag cmd') elif container_variant == 'Sarus+custompull': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' 'load/library/image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' 'image:tag') elif container_variant == 'Sarus+mpi': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' '--mpi image:tag cmd') elif container_variant == 'Singularity': return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' - '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" image:tag cmd') + '-B"/path/two:/two" image:tag cmd') elif container_variant == 'Singularity+cuda': return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' - '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" ' + '-B"/path/two:/two" ' '--nv image:tag cmd') elif container_variant == 'Singularity+nocommand': return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' - '-B"/path/two:/two" -B"${PWD}:/rfm_workdir" image:tag') + '-B"/path/two:/two" image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' "image:tag bash -c 'cd /stagedir;cmd'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' "--mpi image:tag bash -c 'cd /stagedir;cmd'") @@ -126,49 +119,43 @@ def expected_cmd_prepare(container_variant): def expected_cmd_run_opts(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' - '-v "${PWD}":"/rfm_workdir" --foo --bar image:tag cmd') + '--foo --bar image:tag cmd') if container_variant == 'Docker+nocommand': return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' - '-v "${PWD}":"/rfm_workdir" --foo --bar image:tag') + '--foo --bar image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' "--foo --bar image:tag bash -c 'cd /stagedir;cmd'") elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' "--mpi --foo --bar image:tag bash -c 'cd /stagedir;cmd'") elif container_variant in {'Sarus', 'Sarus+nopull'}: return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' '--foo --bar image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' '--foo --bar image:tag') elif container_variant == 'Sarus+mpi': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' '--mpi --foo --bar image:tag cmd') elif container_variant == 'Sarus+custompull': return ('sarus run --workdir="/stagedir" ' '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="${PWD}",destination="/rfm_workdir" ' '--foo --bar load/library/image:tag cmd') elif container_variant == 'Singularity': return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' - '-B"${PWD}:/rfm_workdir" --foo --bar image:tag cmd') + '--foo --bar image:tag cmd') elif container_variant == 'Singularity+cuda': return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' - '-B"${PWD}:/rfm_workdir" --nv --foo --bar image:tag cmd') + '--nv --foo --bar image:tag cmd') elif container_variant == 'Singularity+nocommand': return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' - '-B"${PWD}:/rfm_workdir" --foo --bar image:tag') + '--foo --bar image:tag') def test_mount_points(container_platform, expected_cmd_mount_points): From aae66cca48e630984646c4b2c46802b8e9a73678 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Wed, 17 Feb 2021 17:50:17 +0100 Subject: [PATCH 03/16] Add extra tutorial example --- docs/tutorial_advanced.rst | 3 +- reframe/core/containers.py | 63 ++++----------- .../advanced/containers/container_test.py | 26 ++++-- unittests/test_containers.py | 79 +++++++------------ 4 files changed, 67 insertions(+), 104 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index bafdef0411..aa48d6d394 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -703,11 +703,10 @@ ReFrame will run the container as follows: .. code-block:: console - singularity exec -B"/path/to/test/stagedir:/rfm_stagedir" --workdir="/rfm_stagedir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' + singularity exec -B"/path/to/test/stagedir:/rfm_stagedir" --pwd="/rfm_stagedir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' By default ReFrame will mount the stage directory of the test under ``/rfm_stagedir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. -Users may also change the default working directory of the container using the :attr:`workdir ` attribute: Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: .. code-block:: python diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 14d4c964cf..4d04be6dd3 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -36,18 +36,15 @@ class ContainerPlatform(abc.ABC): #: :default: :class:`None` command = fields.TypedField(str, type(None)) - #: The pull command to be used to pull the container image. + #: Pull the container image before running. #: - #: If an empty string is given as the pull command, then the default - #: pull command of the corresponding container platform is going to be - #: used to pull the image. If set to :class:`None`, then no pull action - #: is going to be performed by the container platform. + #: This does not have any effect for the `Singularity` container platform. #: #: ..versionadded:: 3.4.2 #: - #: :type: :class:`str` or :class:`None` - #: :default: ``''`` - pull_command = fields.TypedField(str, type(None)) + #: :type: :class:`bool` + #: :default: ``True`` + pull_image = fields.TypedField(bool) #: List of mount point pairs for directories to mount inside the container. #: @@ -66,22 +63,12 @@ class ContainerPlatform(abc.ABC): #: :default: ``[]`` options = fields.TypedField(typ.List[str]) - #: The working directory inside the container. - #: - #: If set to :class:`None` then the default working directory of the given - #: container image is used. - #: - #: :type: :class:`str` - #: :default: :class:None - workdir = fields.TypedField(str, type(None)) - def __init__(self): self.image = None self.command = None self.mount_points = [] self.options = [] - self.pull_command = '' - self.workdir = None + self.pull_image = True @abc.abstractmethod def emit_prepare_commands(self): @@ -120,20 +107,14 @@ class Docker(ContainerPlatform): `__.''' def emit_prepare_commands(self): - if self.pull_command == '': - return [f'docker pull {self.image}'] - elif self.pull_command: - return [self.pull_command] - - return [] + return [f'docker pull {self.image}'] if self.pull_image else [] def launch_command(self): super().launch_command() run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] run_opts += self.options - workdir_opt = f'--workdir="{self.workdir}" ' if self.workdir else '' - return (f'docker run --rm {workdir_opt}{" ".join(run_opts)} ' + return (f'docker run --rm {" ".join(run_opts)} ' f'{self.image} {self.command or ""}').rstrip() @@ -152,12 +133,7 @@ def __init__(self): self.with_mpi = False def emit_prepare_commands(self): - if self.pull_command == '': - return [f'sarus pull {self.image}'] - elif self.pull_command: - return [self.pull_command] - - return [] + return [f'sarus pull {self.image}'] if self.pull_image else [] def launch_command(self): super().launch_command() @@ -168,8 +144,7 @@ def launch_command(self): run_opts += self.options - workdir_opt = f'--workdir="{self.workdir}" ' if self.workdir else '' - return (f'sarus run {workdir_opt}{" ".join(run_opts)} {self.image} ' + return (f'sarus run {" ".join(run_opts)} {self.image} ' f'{self.command or ""}').rstrip() @@ -189,12 +164,7 @@ def __init__(self): self.with_mpi = False def emit_prepare_commands(self): - if self.pull_command == '': - return [f'shifter pull {self.image}'] - elif self.pull_command: - return [self.pull_command] - - return [] + return [f'shifter pull {self.image}'] if self.pull_image else [] def launch_command(self): super().launch_command() @@ -204,9 +174,8 @@ def launch_command(self): run_opts.append('--mpi') run_opts += self.options - workdir_opt = f'cd {self.workdir};' if self.workdir else '' - return (f"shifter run {' '.join(run_opts)} {self.image} bash -c '" - f"{workdir_opt}{self.command or ''}'") + return (f"shifter run {' '.join(run_opts)} {self.image} " + f"{self.command or ''}") class Singularity(ContainerPlatform): @@ -233,13 +202,11 @@ def launch_command(self): run_opts.append('--nv') run_opts += self.options - workdir_cmd = f'--workdir="{self.workdir}" ' if self.workdir else '' if self.command: - return (f'singularity exec {workdir_cmd}{" ".join(run_opts)} ' + return (f'singularity exec {" ".join(run_opts)} ' f'{self.image} {self.command}') - return (f'singularity run {workdir_cmd}{" ".join(run_opts)} ' - f'{self.image}') + return f'singularity run {" ".join(run_opts)} {self.image}' class ContainerPlatformField(fields.TypedField): diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index a05224a30c..cb4bc39b7e 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -15,11 +15,27 @@ def __init__(self): self.valid_prog_environs = ['builtin'] self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.command = ("bash -c 'pwd; ls;" - "cat /etc/os-release'") - self.container_platform.workdir = '/rfm_stagedir' + self.container_platform.command = "bash -c 'pwd; cat /etc/os-release'" + self.container_platform.options = ['--pwd=/rfm_stagedir'] self.sanity_patterns = sn.all([ - sn.assert_found(r'^' + self.container_platform.workdir, - self.stdout), + sn.assert_found(r'^/rfm_stagedir', self.stdout), sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', self.stdout), ]) + + +@rfm.simple_test +class ContainerTestWithFile(rfm.RunOnlyRegressionTest): + def __init__(self): + self.descr = 'Run commands inside a container' + self.valid_systems = ['daint:gpu'] + self.valid_prog_environs = ['builtin'] + self.container_platform = 'Singularity' + self.container_platform.image = 'docker://ubuntu:18.04' + self.container_platform.command = ( + "bash -c 'cat /etc/os-release > os_release.txt; " + "cp os_release.txt /rfm_stagedir'" + ) + self.container_platform.options = ['--pwd=/rfm_stagedir'] + self.sanity_patterns = sn.assert_found( + r'18.04.\d+ LTS \(Bionic Beaver\)', 'os_release.txt' + ) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 29ccc74c7b..c6f56b6e7e 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -11,9 +11,8 @@ @pytest.fixture(params=[ 'Docker', 'Docker+nocommand', 'Docker+nopull', 'Sarus', 'Sarus+nocommand', - 'Sarus+nopull', 'Sarus+custompull', 'Sarus+mpi', 'Shifter', 'Shifter+mpi', - 'Singularity', 'Singularity+cuda', 'Singularity+nocommand' -]) + 'Sarus+nopull', 'Sarus+mpi', 'Shifter', 'Shifter+mpi', 'Singularity', + 'Singularity+cuda', 'Singularity+nocommand']) def container_variant(request): return request.param @@ -37,12 +36,7 @@ def container_platform(container_variant): ret.image = 'image:tag' if '+nopull' in container_variant: - ret.pull_command = None - - if container_variant == 'Sarus+custompull': - ret.pull_command = ('docker pull registry/image:tag; ' - 'docker save -o local_image.tar; ' - 'sarus load local_image.tar image:tag') + ret.pull_image = False return ret @@ -50,67 +44,60 @@ def container_platform(container_variant): @pytest.fixture def expected_cmd_mount_points(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: - return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + return ('docker run --rm -v "/path/one":"/one" ' '-v "/path/two":"/two" image:tag cmd') elif container_variant == 'Docker+nocommand': - return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + return ('docker run --rm -v "/path/one":"/one" ' '-v "/path/two":"/two" image:tag') elif container_variant in {'Sarus', 'Sarus+nopull'}: - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' 'image:tag cmd') elif container_variant == 'Sarus+custompull': - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' 'load/library/image:tag cmd') elif container_variant == 'Sarus+nocommand': - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' 'image:tag') elif container_variant == 'Sarus+mpi': - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' '--mpi image:tag cmd') - elif container_variant == 'Singularity': - return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + elif container_variant in {'Singularity', 'Singularity+nopull'}: + return ('singularity exec -B"/path/one:/one" ' '-B"/path/two:/two" image:tag cmd') elif container_variant == 'Singularity+cuda': - return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' - '-B"/path/two:/two" ' - '--nv image:tag cmd') + return ('singularity exec -B"/path/one:/one" ' + '-B"/path/two:/two" --nv image:tag cmd') elif container_variant == 'Singularity+nocommand': - return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' + return ('singularity run -B"/path/one:/one" ' '-B"/path/two:/two" image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "image:tag bash -c 'cd /stagedir;cmd'") + 'image:tag cmd') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' - "--mpi image:tag bash -c 'cd /stagedir;cmd'") + '--mpi image:tag cmd') @pytest.fixture def expected_cmd_prepare(container_variant): - if container_variant in ('Docker', 'Docker+nocommand'): + if container_variant in {'Docker', 'Docker+nocommand'}: return ['docker pull image:tag'] - if container_variant in ('Shifter', 'Shifter+mpi'): + elif container_variant in {'Shifter', 'Shifter+mpi'}: return ['shifter pull image:tag'] - elif container_variant in ('Sarus', 'Sarus+nocommand', 'Sarus+mpi'): + elif container_variant in {'Sarus', 'Sarus+nocommand', 'Sarus+mpi'}: return ['sarus pull image:tag'] - elif container_variant == 'Sarus+custompull': - return [ - 'docker pull registry/image:tag; ' - 'docker save -o local_image.tar; ' - 'sarus load local_image.tar image:tag' - ] else: return [] @@ -118,50 +105,45 @@ def expected_cmd_prepare(container_variant): @pytest.fixture def expected_cmd_run_opts(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: - return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + return ('docker run --rm -v "/path/one":"/one" ' '--foo --bar image:tag cmd') if container_variant == 'Docker+nocommand': - return ('docker run --rm --workdir="/stagedir" -v "/path/one":"/one" ' + return ('docker run --rm -v "/path/one":"/one" ' '--foo --bar image:tag') elif container_variant == 'Shifter': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--foo --bar image:tag bash -c 'cd /stagedir;cmd'") + '--foo --bar image:tag cmd') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' - "--mpi --foo --bar image:tag bash -c 'cd /stagedir;cmd'") + '--mpi --foo --bar image:tag cmd') elif container_variant in {'Sarus', 'Sarus+nopull'}: - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--foo --bar image:tag cmd') elif container_variant == 'Sarus+nocommand': - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--foo --bar image:tag') elif container_variant == 'Sarus+mpi': - return ('sarus run --workdir="/stagedir" ' + return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mpi --foo --bar image:tag cmd') - elif container_variant == 'Sarus+custompull': - return ('sarus run --workdir="/stagedir" ' - '--mount=type=bind,source="/path/one",destination="/one" ' - '--foo --bar load/library/image:tag cmd') - elif container_variant == 'Singularity': - return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + elif container_variant in {'Singularity'}: + return ('singularity exec -B"/path/one:/one" ' '--foo --bar image:tag cmd') elif container_variant == 'Singularity+cuda': - return ('singularity exec --workdir="/stagedir" -B"/path/one:/one" ' + return ('singularity exec -B"/path/one:/one" ' '--nv --foo --bar image:tag cmd') elif container_variant == 'Singularity+nocommand': - return ('singularity run --workdir="/stagedir" -B"/path/one:/one" ' + return ('singularity run -B"/path/one:/one" ' '--foo --bar image:tag') def test_mount_points(container_platform, expected_cmd_mount_points): container_platform.mount_points = [('/path/one', '/one'), ('/path/two', '/two')] - container_platform.workdir = '/stagedir' assert container_platform.launch_command() == expected_cmd_mount_points @@ -177,6 +159,5 @@ def test_prepare_command(container_platform, expected_cmd_prepare): def test_run_opts(container_platform, expected_cmd_run_opts): container_platform.mount_points = [('/path/one', '/one')] - container_platform.workdir = '/stagedir' container_platform.options = ['--foo', '--bar'] assert container_platform.launch_command() == expected_cmd_run_opts From 6ffc9f7b0ed25df03bdbdfb96bb30684ccdc829f Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 22 Feb 2021 14:55:36 +0100 Subject: [PATCH 04/16] Deprecate container fields --- docs/tutorial_advanced.rst | 6 +- reframe/core/containers.py | 82 +++++++++++++++++-- .../advanced/containers/container_test.py | 8 +- unittests/test_pipeline.py | 6 +- 4 files changed, 83 insertions(+), 19 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index aa48d6d394..de98e4f235 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -703,9 +703,9 @@ ReFrame will run the container as follows: .. code-block:: console - singularity exec -B"/path/to/test/stagedir:/rfm_stagedir" --pwd="/rfm_stagedir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' + singularity exec -B"/path/to/test/stagedir:/rfm_workdir" --pwd="/rfm_workdir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' -By default ReFrame will mount the stage directory of the test under ``/rfm_stagedir`` inside the container. +By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: @@ -716,7 +716,7 @@ Besides the stage directory, additional mount points can be specified through th .. tip:: - The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_stagedir`` inside the container where the user can copy directories/files as needed. + The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy directories/files as needed. The aforementioned directories/files will then be available inside the stage directory after the container execution finishes. This is very useful if the above directories/files are going to be used for the sanity/performance checks. If this is the case, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 4d04be6dd3..76a65d9157 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -4,10 +4,12 @@ # SPDX-License-Identifier: BSD-3-Clause import abc +import warnings import reframe.core.fields as fields import reframe.utility.typecheck as typ from reframe.core.exceptions import ContainerError +from reframe.core.warnings import ReframeDeprecationWarning class ContainerPlatform(abc.ABC): @@ -15,7 +17,7 @@ class ContainerPlatform(abc.ABC): #: The default mount location of the test case stage directory inside the #: container - RFM_STAGEDIR = '/rfm_stagedir' + RFM_STAGEDIR = '/rfm_workdir' #: The container image to be used for running the test. #: @@ -36,6 +38,19 @@ class ContainerPlatform(abc.ABC): #: :default: :class:`None` command = fields.TypedField(str, type(None)) + #: The commands to be executed within the container. + #: + #: ..versionchanged:: 3.4.2 + #: The `commands` field is now deprecated. + #: + #: :type: :class:`list[str]` + #: :default: ``[]`` + commands = fields.DeprecatedField( + fields.TypedField(typ.List[str]), + 'The `commands` field is deprecated, please use the `command` field ' + 'to set the command to be executed by the container.', 1 + ) + #: Pull the container image before running. #: #: This does not have any effect for the `Singularity` container platform. @@ -50,7 +65,7 @@ class ContainerPlatform(abc.ABC): #: #: Each mount point is specified as a tuple of #: ``(/path/in/host, /path/in/container)``. The stage directory of the - #: ReFrame test is always mounted under ``/rfm_stagedir`` inside the + #: ReFrame test is always mounted under ``/rfm_workdir`` inside the #: container, independelty of this field. #: #: :type: :class:`list[tuple[str, str]]` @@ -63,9 +78,30 @@ class ContainerPlatform(abc.ABC): #: :default: ``[]`` options = fields.TypedField(typ.List[str]) + #: The working directory of ReFrame inside the container. + #: + #: This is the directory where the test's stage directory is mounted inside + #: the container. This directory is always mounted regardless if + #: :attr:`mount_points` is set or not. + #: + #: ..versionchanged:: 3.4.2 + #: The `commands` field is now deprecated. + #: + #: :type: :class:`str` + #: :default: ``/rfm_workdir`` + workdir = fields.DeprecatedField( + fields.TypedField(str, type(None)), + 'The `workdir` field is deprecated, please use the `options` field to ' + 'set the container working directory', 1 + ) + def __init__(self): self.image = None self.command = None + with warnings.catch_warnings(record=True): + self.commands = [] + self.workdir = self.RFM_STAGEDIR + self.mount_points = [] self.options = [] self.pull_image = True @@ -114,8 +150,19 @@ def launch_command(self): run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] run_opts += self.options - return (f'docker run --rm {" ".join(run_opts)} ' - f'{self.image} {self.command or ""}').rstrip() + run_cmd = 'docker run --rm %s %s bash -c ' % (' '.join(run_opts), + self.image) + + if self.command: + return (f'docker run --rm {" ".join(run_opts)} ' + f'{self.image} {self.command}') + + if self.commands: + return (f"docker run --rm {' '.join(run_opts)} {self.image} " + "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + + return (f'docker run --rm {" ".join(run_opts)} {self.image}') + class Sarus(ContainerPlatform): @@ -144,8 +191,15 @@ def launch_command(self): run_opts += self.options - return (f'sarus run {" ".join(run_opts)} {self.image} ' - f'{self.command or ""}').rstrip() + if self.command: + return (f'sarus run {" ".join(run_opts)} {self.image} ' + f'{self.command}') + + if self.commands: + return (f"sarus run {' '.join(run_opts)} {self.image} " + "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + + return f'sarus run {" ".join(run_opts)} {self.image}' class Shifter(ContainerPlatform): @@ -174,8 +228,16 @@ def launch_command(self): run_opts.append('--mpi') run_opts += self.options - return (f"shifter run {' '.join(run_opts)} {self.image} " - f"{self.command or ''}") + + if self.command: + return (f'shifter run {" ".join(run_opts)} {self.image} ' + f'{self.command}') + + if self.commands: + return (f"shifter run {' '.join(run_opts)} {self.image} " + "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + + return f'shifter run {" ".join(run_opts)} {self.image}' class Singularity(ContainerPlatform): @@ -206,6 +268,10 @@ def launch_command(self): return (f'singularity exec {" ".join(run_opts)} ' f'{self.image} {self.command}') + if self.commands: + return (f"singularity exec {' '.join(run_opts)} {self.image} " + "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + return f'singularity run {" ".join(run_opts)} {self.image}' diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index cb4bc39b7e..495a2c4f11 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -16,9 +16,9 @@ def __init__(self): self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' self.container_platform.command = "bash -c 'pwd; cat /etc/os-release'" - self.container_platform.options = ['--pwd=/rfm_stagedir'] + self.container_platform.options = ['--pwd=/rfm_workdir'] self.sanity_patterns = sn.all([ - sn.assert_found(r'^/rfm_stagedir', self.stdout), + sn.assert_found(r'^/rfm_workdir', self.stdout), sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', self.stdout), ]) @@ -33,9 +33,9 @@ def __init__(self): self.container_platform.image = 'docker://ubuntu:18.04' self.container_platform.command = ( "bash -c 'cat /etc/os-release > os_release.txt; " - "cp os_release.txt /rfm_stagedir'" + "cp os_release.txt /rfm_workdir'" ) - self.container_platform.options = ['--pwd=/rfm_stagedir'] + self.container_platform.options = ['--pwd=/rfm_workdir'] self.sanity_patterns = sn.assert_found( r'18.04.\d+ LTS \(Bionic Beaver\)', 'os_release.txt' ) diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index bdd0611244..c7267f8fcd 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -1117,10 +1117,8 @@ def __init__(self): self.valid_systems = ['*'] self.container_platform = platform self.container_platform.image = image - self.container_platform.commands = [ - 'pwd', 'ls', 'cat /etc/os-release' - ] - self.container_platform.workdir = '/workdir' + self.container_platform.command = ("bash -c 'pwd; ls; " + "cat /etc/os-release'") self.prerun_cmds = ['touch foo'] self.sanity_patterns = sn.all([ sn.assert_found( From 5f729c7321ca547e20db1df46ba11397a2369128 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Mon, 22 Feb 2021 15:55:55 +0100 Subject: [PATCH 05/16] Remove unused imports and fix unittests --- reframe/core/containers.py | 2 -- unittests/test_pipeline.py | 28 +++++++++++++++------------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 76a65d9157..62bf15e51c 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -9,7 +9,6 @@ import reframe.core.fields as fields import reframe.utility.typecheck as typ from reframe.core.exceptions import ContainerError -from reframe.core.warnings import ReframeDeprecationWarning class ContainerPlatform(abc.ABC): @@ -164,7 +163,6 @@ def launch_command(self): return (f'docker run --rm {" ".join(run_opts)} {self.image}') - class Sarus(ContainerPlatform): '''Container platform backend for running containers with `Sarus `__.''' diff --git a/unittests/test_pipeline.py b/unittests/test_pipeline.py index c7267f8fcd..65431b8cc9 100644 --- a/unittests/test_pipeline.py +++ b/unittests/test_pipeline.py @@ -85,8 +85,9 @@ def local_user_exec_ctx(user_system): try: environ = partition.environs[0] except IndexError: - pytest.skip('no environments configured for partition: %s' % - partition.fullname) + pytest.skip( + f'no environments configured for partition: {partition.fullname}' + ) yield partition, environ @@ -100,8 +101,9 @@ def remote_exec_ctx(user_system): try: environ = partition.environs[0] except IndexError: - pytest.skip('no environments configured for partition: %s' % - partition.fullname) + pytest.skip( + f'no environments configured for partition: {partition.fullname}' + ) yield partition, environ @@ -111,7 +113,7 @@ def container_remote_exec_ctx(remote_exec_ctx): def _container_exec_ctx(platform): partition = remote_exec_ctx[0] if platform not in partition.container_environs.keys(): - pytest.skip('%s is not configured on the system' % platform) + pytest.skip(f'{platform} is not configured on the system') yield from remote_exec_ctx @@ -123,7 +125,7 @@ def container_local_exec_ctx(local_user_exec_ctx): def _container_exec_ctx(platform): partition = local_user_exec_ctx[0] if platform not in partition.container_environs.keys(): - pytest.skip('%s is not configured on the system' % platform) + pytest.skip(f'{platform} is not configured on the system') yield from local_user_exec_ctx @@ -474,7 +476,7 @@ def set_resources(self): _run(test, partition, environ) expected_job_options = {'--gres=gpu:2', '#DW jobdw capacity=100GB', - '#DW stage_in source=%s' % test.stagedir, + f'#DW stage_in source={test.stagedir}', '--foo'} assert expected_job_options == set(test.job.options) @@ -756,7 +758,7 @@ def __init__(self, a): self.a = a def __repr__(self): - return 'C(%s)' % self.a + return f'C({self.a})' class MyTest(rfm.RegressionTest): def __init__(self, a, b): @@ -1081,7 +1083,7 @@ def extract_perf(patt, tag): sn.extractsingle(patt, perf_file, tag, float) ) with open('perf.log', 'a') as fp: - fp.write('%s=%s' % (tag, val)) + fp.write(f'{tag}={val}') return val @@ -1117,12 +1119,12 @@ def __init__(self): self.valid_systems = ['*'] self.container_platform = platform self.container_platform.image = image - self.container_platform.command = ("bash -c 'pwd; ls; " - "cat /etc/os-release'") + self.container_platform.command = ( + "bash -c 'cd /rfm_workdir; pwd; ls; cat /etc/os-release'" + ) self.prerun_cmds = ['touch foo'] self.sanity_patterns = sn.all([ - sn.assert_found( - r'^' + self.container_platform.workdir, self.stdout), + sn.assert_found(r'^/rfm_workdir', self.stdout), sn.assert_found(r'^foo', self.stdout), sn.assert_found( r'18\.04\.\d+ LTS \(Bionic Beaver\)', self.stdout), From 0369c9ff49931689eee85aacd316290716689b57 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Tue, 23 Feb 2021 17:09:00 +0100 Subject: [PATCH 06/16] Test deprecated container platform fields --- docs/tutorial_advanced.rst | 23 ++-- reframe/core/containers.py | 32 +++-- .../advanced/containers/container_test.py | 24 +--- unittests/test_containers.py | 129 +++++++++++++++--- 4 files changed, 150 insertions(+), 58 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index de98e4f235..7905b88910 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -686,7 +686,7 @@ The following test will use a Singularity container to run: .. literalinclude:: ../tutorials/advanced/containers/container_test.py :lines: 6- - :emphasize-lines: 11-15 + :emphasize-lines: 11-16 A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform ` attribute. This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. @@ -694,7 +694,7 @@ In this case, the test will be using `Singularity `__ as a co If such a platform is not `configured `__ for the current system, the test will fail. As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image `. -The default command that the container runs can be overwritten by setting the :attr:`command ` of the container platform. The :attr:`image ` is the only mandatory attribute for container-based checks. It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. @@ -703,7 +703,7 @@ ReFrame will run the container as follows: .. code-block:: console - singularity exec -B"/path/to/test/stagedir:/rfm_workdir" --pwd="/rfm_workdir" docker://ubuntu:18.04 bash -c 'pwd; ls; cat /etc/os-release' + singularity exec -B"/path/to/test/stagedir:/rfm_workdir" --pwd="/rfm_workdir" docker://ubuntu:18.04 bash -c 'pwd; cat /etc/os-release > release.txt' By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. @@ -714,12 +714,19 @@ Besides the stage directory, additional mount points can be specified through th self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), ('/path/to/host/dir2', '/path/to/container/mount_point2')] -.. tip:: - The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy directories/files as needed. - The aforementioned directories/files will then be available inside the stage directory after the container execution finishes. - This is very useful if the above directories/files are going to be used for the sanity/performance checks. - If this is the case, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. +The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy directories/files as needed. +The aforementioned directories/files will then be available inside the stage directory after the container execution finishes. +This is very useful if the above directories/files are going to be used for the sanity/performance checks. +If this is the case, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. +In this test, we have changed the directory that the container is going to run the commands using the `--pwd=/rfm_workdir` through the :attr:`options `. +Since this is the directory that ReFrame mounts the stage directory inside the container, the files created there are available after the container execution finishes. +Therefore, the ``release.txt`` where the output of the ``cat /etc/os-release`` is redirected, can be used in the subsequent sanity function: + +.. code-block:: python + + sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt') + For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`. diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 62bf15e51c..e1417d8f0d 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -29,7 +29,7 @@ class ContainerPlatform(abc.ABC): #: If no command is given, then the default command of the corresponding #: container image is going to be executed. #: - #: ..versionchanged:: 3.4.2 + #: ..versionchanged:: 3.5 #: Changed the attribute name from `commands` to `command` and its type #: to a string. #: @@ -39,7 +39,7 @@ class ContainerPlatform(abc.ABC): #: The commands to be executed within the container. #: - #: ..versionchanged:: 3.4.2 + #: ..versionchanged:: 3.5 #: The `commands` field is now deprecated. #: #: :type: :class:`list[str]` @@ -54,7 +54,7 @@ class ContainerPlatform(abc.ABC): #: #: This does not have any effect for the `Singularity` container platform. #: - #: ..versionadded:: 3.4.2 + #: ..versionadded:: 3.5 #: #: :type: :class:`bool` #: :default: ``True`` @@ -83,7 +83,7 @@ class ContainerPlatform(abc.ABC): #: the container. This directory is always mounted regardless if #: :attr:`mount_points` is set or not. #: - #: ..versionchanged:: 3.4.2 + #: ..versionchanged:: 3.5 #: The `commands` field is now deprecated. #: #: :type: :class:`str` @@ -158,7 +158,7 @@ def launch_command(self): if self.commands: return (f"docker run --rm {' '.join(run_opts)} {self.image} " - "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") return (f'docker run --rm {" ".join(run_opts)} {self.image}') @@ -178,7 +178,13 @@ def __init__(self): self.with_mpi = False def emit_prepare_commands(self): - return [f'sarus pull {self.image}'] if self.pull_image else [] + # The format that Sarus uses to call the images is + # //:. If an image was loaded + # locally from a tar file, the is 'load'. + if not self.pull_image or self.image.startswith('load/'): + return [] + else: + return [f'sarus pull {self.image}'] def launch_command(self): super().launch_command() @@ -195,7 +201,7 @@ def launch_command(self): if self.commands: return (f"sarus run {' '.join(run_opts)} {self.image} " - "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") return f'sarus run {" ".join(run_opts)} {self.image}' @@ -216,7 +222,13 @@ def __init__(self): self.with_mpi = False def emit_prepare_commands(self): - return [f'shifter pull {self.image}'] if self.pull_image else [] + # The format that Shifter uses to call the images is + # //:. If an image was loaded + # locally from a tar file, the is 'load'. + if not self.pull_image or self.image.startswith('load/'): + return [] + else: + return [f'shifter pull {self.image}'] def launch_command(self): super().launch_command() @@ -233,7 +245,7 @@ def launch_command(self): if self.commands: return (f"shifter run {' '.join(run_opts)} {self.image} " - "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") return f'shifter run {" ".join(run_opts)} {self.image}' @@ -268,7 +280,7 @@ def launch_command(self): if self.commands: return (f"singularity exec {' '.join(run_opts)} {self.image} " - "bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") + f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") return f'singularity run {" ".join(run_opts)} {self.image}' diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index 495a2c4f11..df44e0f78f 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -15,27 +15,11 @@ def __init__(self): self.valid_prog_environs = ['builtin'] self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.command = "bash -c 'pwd; cat /etc/os-release'" + self.container_platform.command = ( + "bash -c 'pwd; cat /etc/os-release > release.txt'" + ) self.container_platform.options = ['--pwd=/rfm_workdir'] self.sanity_patterns = sn.all([ sn.assert_found(r'^/rfm_workdir', self.stdout), - sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', self.stdout), + sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt') ]) - - -@rfm.simple_test -class ContainerTestWithFile(rfm.RunOnlyRegressionTest): - def __init__(self): - self.descr = 'Run commands inside a container' - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['builtin'] - self.container_platform = 'Singularity' - self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.command = ( - "bash -c 'cat /etc/os-release > os_release.txt; " - "cp os_release.txt /rfm_workdir'" - ) - self.container_platform.options = ['--pwd=/rfm_workdir'] - self.sanity_patterns = sn.assert_found( - r'18.04.\d+ LTS \(Bionic Beaver\)', 'os_release.txt' - ) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index c6f56b6e7e..2f954fd853 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -6,13 +6,15 @@ import pytest import reframe.core.containers as containers +import reframe.core.warnings as warn from reframe.core.exceptions import ContainerError @pytest.fixture(params=[ 'Docker', 'Docker+nocommand', 'Docker+nopull', 'Sarus', 'Sarus+nocommand', - 'Sarus+nopull', 'Sarus+mpi', 'Shifter', 'Shifter+mpi', 'Singularity', - 'Singularity+cuda', 'Singularity+nocommand']) + 'Sarus+nopull', 'Sarus+mpi', 'Sarus+load', 'Shifter', 'Shifter+nocommand', + 'Shifter+mpi', 'Shifter+nopull', 'Shifter+load', 'Singularity', + 'Singularity+nocommand', 'Singularity+cuda']) def container_variant(request): return request.param @@ -30,7 +32,7 @@ def container_platform(container_variant): if '+cuda' in container_variant: ret.with_cuda = True - if container_variant == 'Sarus+custompull': + if '+load' in container_variant: ret.image = 'load/library/image:tag' else: ret.image = 'image:tag' @@ -54,11 +56,6 @@ def expected_cmd_mount_points(container_variant): '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' 'image:tag cmd') - elif container_variant == 'Sarus+custompull': - return ('sarus run ' - '--mount=type=bind,source="/path/one",destination="/one" ' - '--mount=type=bind,source="/path/two",destination="/two" ' - 'load/library/image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' @@ -69,32 +66,47 @@ def expected_cmd_mount_points(container_variant): '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' '--mpi image:tag cmd') - elif container_variant in {'Singularity', 'Singularity+nopull'}: - return ('singularity exec -B"/path/one:/one" ' - '-B"/path/two:/two" image:tag cmd') - elif container_variant == 'Singularity+cuda': - return ('singularity exec -B"/path/one:/one" ' - '-B"/path/two:/two" --nv image:tag cmd') - elif container_variant == 'Singularity+nocommand': - return ('singularity run -B"/path/one:/one" ' - '-B"/path/two:/two" image:tag') - elif container_variant == 'Shifter': + elif container_variant == 'Sarus+load': + return ('sarus run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/path/two",destination="/two" ' + 'load/library/image:tag cmd') + elif container_variant in {'Shifter', 'Shifter+nopull'}: return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' 'image:tag cmd') + elif container_variant == 'Shifter+nocommand': + return ('shifter run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/path/two",destination="/two" ' + 'image:tag') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' '--mpi image:tag cmd') + elif container_variant == 'Shifter+load': + return ('shifter run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/path/two",destination="/two" ' + 'load/library/image:tag cmd') + elif container_variant in {'Singularity', 'Singularity+nopull'}: + return ('singularity exec -B"/path/one:/one" ' + '-B"/path/two:/two" image:tag cmd') + elif container_variant == 'Singularity+cuda': + return ('singularity exec -B"/path/one:/one" ' + '-B"/path/two:/two" --nv image:tag cmd') + elif container_variant == 'Singularity+nocommand': + return ('singularity run -B"/path/one:/one" ' + '-B"/path/two:/two" image:tag') @pytest.fixture def expected_cmd_prepare(container_variant): if container_variant in {'Docker', 'Docker+nocommand'}: return ['docker pull image:tag'] - elif container_variant in {'Shifter', 'Shifter+mpi'}: + elif container_variant in {'Shifter', 'Shifter+nocommand', 'Shifter+mpi'}: return ['shifter pull image:tag'] elif container_variant in {'Sarus', 'Sarus+nocommand', 'Sarus+mpi'}: return ['sarus pull image:tag'] @@ -110,18 +122,34 @@ def expected_cmd_run_opts(container_variant): if container_variant == 'Docker+nocommand': return ('docker run --rm -v "/path/one":"/one" ' '--foo --bar image:tag') - elif container_variant == 'Shifter': + elif container_variant in {'Shifter', 'Shifter+nopull'}: return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--foo --bar image:tag cmd') + elif container_variant == 'Shifter+nocommand': + return ('shifter run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--foo --bar image:tag') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mpi --foo --bar image:tag cmd') + elif container_variant == 'Shifter+load': + return ('shifter run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--foo --bar load/library/image:tag cmd') elif container_variant in {'Sarus', 'Sarus+nopull'}: return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--foo --bar image:tag cmd') + elif container_variant == 'Sarus': + return ('sarus run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--foo --bar image:tag cmd') + elif container_variant == 'Sarus+load': + return ('sarus run ' + '--mount=type=bind,source="/path/one",destination="/one" ' + '--foo --bar load/library/image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' @@ -161,3 +189,64 @@ def test_run_opts(container_platform, expected_cmd_run_opts): container_platform.mount_points = [('/path/one', '/one')] container_platform.options = ['--foo', '--bar'] assert container_platform.launch_command() == expected_cmd_run_opts + + +@pytest.fixture(params=[ + 'Docker', 'Singularity', 'Sarus', 'Shifter']) +def container_variant_deprecated(request): + return request.param + + +@pytest.fixture +def platform_deprecated(container_variant_deprecated): + ret = containers.__dict__[container_variant_deprecated]() + ret.image = 'image:tag' + ret.options = ['--foo'] + return ret + + +@pytest.fixture +def expected_run_commands(container_variant_deprecated): + if container_variant_deprecated == 'Docker': + return ("docker run --rm --foo image:tag bash -c 'cd /rfm_workdir; " + "cmd1; cmd2'") + elif container_variant_deprecated == 'Sarus': + return ("sarus run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " + "cmd2'") + elif container_variant_deprecated == 'Shifter': + return ("shifter run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " + "cmd2'") + elif container_variant_deprecated == 'Singularity': + return ("singularity exec --foo image:tag bash -c 'cd /rfm_workdir; " + "cmd1; cmd2'") + + +@pytest.fixture +def expected_run_workdir(container_variant_deprecated): + if container_variant_deprecated == 'Docker': + return ("docker run --rm --foo image:tag bash -c 'cd foodir; cmd1; " + "cmd2'") + elif container_variant_deprecated == 'Sarus': + return "sarus run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + elif container_variant_deprecated == 'Shifter': + return "shifter run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + elif container_variant_deprecated == 'Singularity': + return ("singularity exec --foo image:tag bash -c 'cd foodir; cmd1; " + "cmd2'") + + +def test_run_commands(platform_deprecated, expected_run_commands): + with pytest.warns(warn.ReframeDeprecationWarning): + platform_deprecated.commands = ['cmd1', 'cmd2'] + + assert platform_deprecated.launch_command() == expected_run_commands + + +def test_run_workdir(platform_deprecated, expected_run_workdir): + with pytest.warns(warn.ReframeDeprecationWarning): + platform_deprecated.commands = ['cmd1', 'cmd2'] + + with pytest.warns(warn.ReframeDeprecationWarning): + platform_deprecated.workdir = 'foodir' + + assert platform_deprecated.launch_command() == expected_run_workdir From ee88b01f3e03cff8fee990ab55e4e9756d2f1107 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 11:47:26 +0100 Subject: [PATCH 07/16] Further fixes in container platforms --- docs/tutorial_advanced.rst | 26 +++--- reframe/core/containers.py | 82 ++++++------------- .../advanced/containers/container_test.py | 10 +-- unittests/test_containers.py | 19 ++--- 4 files changed, 54 insertions(+), 83 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 7905b88910..9d26d074b8 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -686,7 +686,7 @@ The following test will use a Singularity container to run: .. literalinclude:: ../tutorials/advanced/containers/container_test.py :lines: 6- - :emphasize-lines: 11-16 + :emphasize-lines: 11-15 A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform ` attribute. This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. @@ -694,7 +694,7 @@ In this case, the test will be using `Singularity `__ as a co If such a platform is not `configured `__ for the current system, the test will fail. As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image `. -The default command that the container runs can be overwritten by setting the :attr:`command ` of the container platform. +The default command that the container runs can be overwritten by setting the :attr:`command ` attribute of the container platform. The :attr:`image ` is the only mandatory attribute for container-based checks. It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. @@ -703,7 +703,7 @@ ReFrame will run the container as follows: .. code-block:: console - singularity exec -B"/path/to/test/stagedir:/rfm_workdir" --pwd="/rfm_workdir" docker://ubuntu:18.04 bash -c 'pwd; cat /etc/os-release > release.txt' + singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. @@ -715,13 +715,19 @@ Besides the stage directory, additional mount points can be specified through th ('/path/to/host/dir2', '/path/to/container/mount_point2')] -The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy directories/files as needed. -The aforementioned directories/files will then be available inside the stage directory after the container execution finishes. -This is very useful if the above directories/files are going to be used for the sanity/performance checks. -If this is the case, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. -In this test, we have changed the directory that the container is going to run the commands using the `--pwd=/rfm_workdir` through the :attr:`options `. -Since this is the directory that ReFrame mounts the stage directory inside the container, the files created there are available after the container execution finishes. -Therefore, the ``release.txt`` where the output of the ``cat /etc/os-release`` is redirected, can be used in the subsequent sanity function: +The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy artifacts as needed. +The aforementioned artifacts will then be available inside the stage directory after the container execution finishes. +This is very useful if the above artifacts are going to be used for the sanity/performance checks. +If the copy is not performed by the default container commands, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. +In the current test, the output of the ``cat /etc/os-release`` is made available outside the container since we have used the command: + +.. code-block:: bash + + bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' + + +and ``/rfm_workdir`` corresponds to the stage directory on the host system. +Therefore, the ``release.txt`` file can now be used in the subsequent sanity function: .. code-block:: python diff --git a/reframe/core/containers.py b/reframe/core/containers.py index cee07102c7..53a1c4ac06 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD-3-Clause import abc -import warnings import reframe.core.fields as fields import reframe.utility.typecheck as typ @@ -29,7 +28,7 @@ class ContainerPlatform(abc.ABC): #: If no command is given, then the default command of the corresponding #: container image is going to be executed. #: - #: ..versionchanged:: 3.5 + #: ..versionadded:: 3.5.0 #: Changed the attribute name from `commands` to `command` and its type #: to a string. #: @@ -37,17 +36,19 @@ class ContainerPlatform(abc.ABC): #: :default: :class:`None` command = fields.TypedField(str, type(None)) + _commands = fields.TypedField(typ.List[str]) #: The commands to be executed within the container. #: - #: ..versionchanged:: 3.5 - #: The `commands` field is now deprecated. + #: ..deprecated:: 3.5.0 + #: Please use the `command` field instead. #: #: :type: :class:`list[str]` #: :default: ``[]`` commands = fields.DeprecatedField( - fields.TypedField(typ.List[str]), + _commands, 'The `commands` field is deprecated, please use the `command` field ' - 'to set the command to be executed by the container.', 1 + 'to set the command to be executed by the container.', + fields.DeprecatedField.OP_SET, ) #: Pull the container image before running. @@ -77,29 +78,32 @@ class ContainerPlatform(abc.ABC): #: :default: ``[]`` options = fields.TypedField(typ.List[str]) + _workdir = fields.TypedField(str, type(None)) #: The working directory of ReFrame inside the container. #: #: This is the directory where the test's stage directory is mounted inside #: the container. This directory is always mounted regardless if #: :attr:`mount_points` is set or not. #: - #: ..versionchanged:: 3.5 - #: The `commands` field is now deprecated. + #: ..deprecated:: 3.5 + #: Please use the `options` field to set the working directory. #: #: :type: :class:`str` #: :default: ``/rfm_workdir`` workdir = fields.DeprecatedField( - fields.TypedField(str, type(None)), + _workdir, 'The `workdir` field is deprecated, please use the `options` field to ' - 'set the container working directory', 1 + 'set the container working directory', + fields.DeprecatedField.OP_SET, ) def __init__(self): self.image = None self.command = None - with warnings.catch_warnings(record=True): - self.commands = [] - self.workdir = self.RFM_STAGEDIR + + # NOTE: Here we set the target fields directly to avoid the warnings + self._commands = [] + self._workdir = self.RFM_STAGEDIR self.mount_points = [] self.options = [] @@ -155,9 +159,6 @@ def launch_command(self): run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] run_opts += self.options - run_cmd = 'docker run --rm %s %s bash -c ' % (' '.join(run_opts), - self.image) - if self.command: return (f'docker run --rm {" ".join(run_opts)} ' f'{self.image} {self.command}') @@ -166,7 +167,7 @@ def launch_command(self): return (f"docker run --rm {' '.join(run_opts)} {self.image} " f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") - return (f'docker run --rm {" ".join(run_opts)} {self.image}') + return f'docker run --rm {" ".join(run_opts)} {self.image}' class Sarus(ContainerPlatform): @@ -182,6 +183,7 @@ class Sarus(ContainerPlatform): def __init__(self): super().__init__() self.with_mpi = False + self._command = 'sarus' def emit_prepare_commands(self): # The format that Sarus uses to call the images is @@ -190,7 +192,7 @@ def emit_prepare_commands(self): if not self.pull_image or self.image.startswith('load/'): return [] else: - return [f'sarus pull {self.image}'] + return [f'{self._command} pull {self.image}'] def launch_command(self): super().launch_command() @@ -202,58 +204,24 @@ def launch_command(self): run_opts += self.options if self.command: - return (f'sarus run {" ".join(run_opts)} {self.image} ' + return (f'{self._command} run {" ".join(run_opts)} {self.image} ' f'{self.command}') if self.commands: - return (f"sarus run {' '.join(run_opts)} {self.image} " + return (f"{self._command} run {' '.join(run_opts)} {self.image} " f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") - return f'sarus run {" ".join(run_opts)} {self.image}' + return f'{self._command} run {" ".join(run_opts)} {self.image}' -class Shifter(ContainerPlatform): +class Shifter(Sarus): '''Container platform backend for running containers with `Shifter `__. ''' - #: Enable MPI support when launching the container. - #: - #: :type: boolean - #: :default: :class:`False` - with_mpi = fields.TypedField(bool) - def __init__(self): super().__init__() - self.with_mpi = False - - def emit_prepare_commands(self): - # The format that Shifter uses to call the images is - # //:. If an image was loaded - # locally from a tar file, the is 'load'. - if not self.pull_image or self.image.startswith('load/'): - return [] - else: - return [f'shifter pull {self.image}'] - - def launch_command(self): - super().launch_command() - run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' - for mp in self.mount_points] - if self.with_mpi: - run_opts.append('--mpi') - - run_opts += self.options - - if self.command: - return (f'shifter run {" ".join(run_opts)} {self.image} ' - f'{self.command}') - - if self.commands: - return (f"shifter run {' '.join(run_opts)} {self.image} " - f"bash -c 'cd {self.workdir}; {'; '.join(self.commands)}'") - - return f'shifter run {" ".join(run_opts)} {self.image}' + self._command = 'shifter' class Singularity(ContainerPlatform): diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index df44e0f78f..8658428fb5 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -16,10 +16,8 @@ def __init__(self): self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' self.container_platform.command = ( - "bash -c 'pwd; cat /etc/os-release > release.txt'" + "bash -c 'cat /etc/os-release | tee release.txt'" + ) + self.sanity_patterns = sn.assert_found( + r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt' ) - self.container_platform.options = ['--pwd=/rfm_workdir'] - self.sanity_patterns = sn.all([ - sn.assert_found(r'^/rfm_workdir', self.stdout), - sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt') - ]) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 2f954fd853..fed3f0f5f3 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -11,10 +11,12 @@ @pytest.fixture(params=[ - 'Docker', 'Docker+nocommand', 'Docker+nopull', 'Sarus', 'Sarus+nocommand', - 'Sarus+nopull', 'Sarus+mpi', 'Sarus+load', 'Shifter', 'Shifter+nocommand', - 'Shifter+mpi', 'Shifter+nopull', 'Shifter+load', 'Singularity', - 'Singularity+nocommand', 'Singularity+cuda']) + 'Docker', 'Docker+nocommand', 'Docker+nopull', + 'Sarus', 'Sarus+nocommand', 'Sarus+nopull', 'Sarus+mpi', 'Sarus+load', + 'Shifter', 'Shifter+nocommand', 'Shifter+mpi', 'Shifter+nopull', + 'Shifter+load', + 'Singularity', 'Singularity+nocommand', 'Singularity+cuda' +]) def container_variant(request): return request.param @@ -159,14 +161,12 @@ def expected_cmd_run_opts(container_variant): '--mount=type=bind,source="/path/one",destination="/one" ' '--mpi --foo --bar image:tag cmd') elif container_variant in {'Singularity'}: - return ('singularity exec -B"/path/one:/one" ' - '--foo --bar image:tag cmd') + return 'singularity exec -B"/path/one:/one" --foo --bar image:tag cmd' elif container_variant == 'Singularity+cuda': return ('singularity exec -B"/path/one:/one" ' '--nv --foo --bar image:tag cmd') elif container_variant == 'Singularity+nocommand': - return ('singularity run -B"/path/one:/one" ' - '--foo --bar image:tag') + return 'singularity run -B"/path/one:/one" --foo --bar image:tag' def test_mount_points(container_platform, expected_cmd_mount_points): @@ -191,8 +191,7 @@ def test_run_opts(container_platform, expected_cmd_run_opts): assert container_platform.launch_command() == expected_cmd_run_opts -@pytest.fixture(params=[ - 'Docker', 'Singularity', 'Sarus', 'Shifter']) +@pytest.fixture(params=['Docker', 'Singularity', 'Sarus', 'Shifter']) def container_variant_deprecated(request): return request.param From 7b289bd5b081b6ea67f45b066e7892e0fbbc2043 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 14:07:37 +0100 Subject: [PATCH 08/16] Address most of the PR comments --- reframe/core/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 53a1c4ac06..45b05bff13 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -48,7 +48,7 @@ class ContainerPlatform(abc.ABC): _commands, 'The `commands` field is deprecated, please use the `command` field ' 'to set the command to be executed by the container.', - fields.DeprecatedField.OP_SET, + fields.DeprecatedField.OP_SET, from_version='3.5.0' ) #: Pull the container image before running. @@ -94,7 +94,7 @@ class ContainerPlatform(abc.ABC): _workdir, 'The `workdir` field is deprecated, please use the `options` field to ' 'set the container working directory', - fields.DeprecatedField.OP_SET, + fields.DeprecatedField.OP_SET, from_version='3.5.0' ) def __init__(self): From b4c971d4018a44ee26dcb957b8443dafb8ba7047 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 14:14:44 +0100 Subject: [PATCH 09/16] Fix PEP8 issues --- reframe/core/containers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 45b05bff13..f3adabd56c 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -48,7 +48,7 @@ class ContainerPlatform(abc.ABC): _commands, 'The `commands` field is deprecated, please use the `command` field ' 'to set the command to be executed by the container.', - fields.DeprecatedField.OP_SET, from_version='3.5.0' + fields.DeprecatedField.OP_SET, from_version='3.5.0' ) #: Pull the container image before running. @@ -94,7 +94,7 @@ class ContainerPlatform(abc.ABC): _workdir, 'The `workdir` field is deprecated, please use the `options` field to ' 'set the container working directory', - fields.DeprecatedField.OP_SET, from_version='3.5.0' + fields.DeprecatedField.OP_SET, from_version='3.5.0' ) def __init__(self): From 326e8fe716a5821ed81f6f1dadebc4897d36d284 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 14:22:36 +0100 Subject: [PATCH 10/16] Fix command in container tutorial --- tutorials/advanced/containers/container_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index 8658428fb5..87d5e4121b 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -16,7 +16,7 @@ def __init__(self): self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' self.container_platform.command = ( - "bash -c 'cat /etc/os-release | tee release.txt'" + "bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'" ) self.sanity_patterns = sn.assert_found( r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt' From 24e1079c41e8eabb74d6ecee0de9579c5fe39c53 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 16:05:51 +0100 Subject: [PATCH 11/16] Address PR comments (version 2) --- docs/tutorial_advanced.rst | 16 ++++++++----- reframe/core/containers.py | 23 +++++++++++-------- reframe/core/pipeline.py | 4 ++-- .../advanced/containers/container_test.py | 8 ++++--- 4 files changed, 30 insertions(+), 21 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 9d26d074b8..9e48ab5ab7 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -716,10 +716,10 @@ Besides the stage directory, additional mount points can be specified through th The container filesystem is ephemeral, therefore, ReFrame mounts the stage directory under ``/rfm_workdir`` inside the container where the user can copy artifacts as needed. -The aforementioned artifacts will then be available inside the stage directory after the container execution finishes. -This is very useful if the above artifacts are going to be used for the sanity/performance checks. -If the copy is not performed by the default container commands, the user should overwrite the default command executed by the container, using the :attr:`command ` to include the appropriate copy commands. -In the current test, the output of the ``cat /etc/os-release`` is made available outside the container since we have used the command: +These artifacts will therefore be available inside the stage directory after the container execution finishes. +This is very useful if the artifacts are needed for the sanity or performance checks. +If the copy is not performed by the default container command, the user can override this command by settings the :attr:`command ` attribute such as to include the appropriate copy commands. +In the current test, the output of the ``cat /etc/os-release`` is available both in the standard output as well as in the ``release.txt`` file, since we have used the command: .. code-block:: bash @@ -727,11 +727,15 @@ In the current test, the output of the ``cat /etc/os-release`` is made available and ``/rfm_workdir`` corresponds to the stage directory on the host system. -Therefore, the ``release.txt`` file can now be used in the subsequent sanity function: +Therefore, the ``release.txt`` file can now be used in the subsequent sanity checks: .. code-block:: python - sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt') + os_release_pattern = r'18.04.\d+ LTS \(Bionic Beaver\)' + self.sanity_patterns = sn.all([ + sn.assert_found(os_release_pattern, 'release.txt'), + sn.assert_found(os_release_pattern, self.stdout) + ]) For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. diff --git a/reframe/core/containers.py b/reframe/core/containers.py index f3adabd56c..f4d7a77c03 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -10,12 +10,14 @@ from reframe.core.exceptions import ContainerError +_STAGEDIR_MOUNT = '/rfm_workdir' + + class ContainerPlatform(abc.ABC): '''The abstract base class of any container platform.''' #: The default mount location of the test case stage directory inside the #: container - RFM_STAGEDIR = '/rfm_workdir' #: The container image to be used for running the test. #: @@ -28,9 +30,9 @@ class ContainerPlatform(abc.ABC): #: If no command is given, then the default command of the corresponding #: container image is going to be executed. #: - #: ..versionadded:: 3.5.0 - #: Changed the attribute name from `commands` to `command` and its type - #: to a string. + #: .. versionadded:: 3.5.0 + #: Changed the attribute name from `commands` to `command` and its type + #: to a string. #: #: :type: :class:`str` or :class:`None` #: :default: :class:`None` @@ -39,8 +41,8 @@ class ContainerPlatform(abc.ABC): _commands = fields.TypedField(typ.List[str]) #: The commands to be executed within the container. #: - #: ..deprecated:: 3.5.0 - #: Please use the `command` field instead. + #: .. deprecated:: 3.5.0 + #: Please use the `command` field instead. #: #: :type: :class:`list[str]` #: :default: ``[]`` @@ -55,7 +57,7 @@ class ContainerPlatform(abc.ABC): #: #: This does not have any effect for the `Singularity` container platform. #: - #: ..versionadded:: 3.5 + #: .. versionadded:: 3.5 #: #: :type: :class:`bool` #: :default: ``True`` @@ -85,8 +87,8 @@ class ContainerPlatform(abc.ABC): #: the container. This directory is always mounted regardless if #: :attr:`mount_points` is set or not. #: - #: ..deprecated:: 3.5 - #: Please use the `options` field to set the working directory. + #: .. deprecated:: 3.5 + #: Please use the `options` field to set the working directory. #: #: :type: :class:`str` #: :default: ``/rfm_workdir`` @@ -101,7 +103,8 @@ def __init__(self): self.image = None self.command = None - # NOTE: Here we set the target fields directly to avoid the warnings + # NOTE: Here we set the target fields directly to avoid the deprecation + # warnings self._commands = [] self._workdir = self.RFM_STAGEDIR diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ec1b579033..03191720b4 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -33,7 +33,7 @@ import reframe.utility.udeps as udeps from reframe.core.backends import getlauncher, getscheduler from reframe.core.buildsystems import BuildSystemField -from reframe.core.containers import ContainerPlatformField +from reframe.core.containers import _STAGEDIR_MOUNT, ContainerPlatformField from reframe.core.deferrable import _DeferredExpression from reframe.core.exceptions import (BuildError, DependencyError, PipelineError, SanityError, @@ -1290,7 +1290,7 @@ def run(self): self.container_platform.validate() self.container_platform.mount_points += [ - (self._stagedir, self.container_platform.RFM_STAGEDIR) + (self._stagedir, _STAGEDIR_MOUNT) ] # We replace executable and executable_opts in case of containers diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index 87d5e4121b..c440c6c0a4 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -18,6 +18,8 @@ def __init__(self): self.container_platform.command = ( "bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'" ) - self.sanity_patterns = sn.assert_found( - r'18.04.\d+ LTS \(Bionic Beaver\)', 'release.txt' - ) + os_release_pattern = r'18.04.\d+ LTS \(Bionic Beaver\)' + self.sanity_patterns = sn.all([ + sn.assert_found(os_release_pattern, 'release.txt'), + sn.assert_found(os_release_pattern, self.stdout) + ]) From d952ff49919884d9ef95ede217450351c7d9923a Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 16:10:55 +0100 Subject: [PATCH 12/16] Use _STAGEDIR_MOUNT in the container platforms --- reframe/core/containers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index f4d7a77c03..f951ac51f2 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -106,7 +106,7 @@ def __init__(self): # NOTE: Here we set the target fields directly to avoid the deprecation # warnings self._commands = [] - self._workdir = self.RFM_STAGEDIR + self._workdir = _STAGEDIR_MOUNT self.mount_points = [] self.options = [] From ba07fb64ee8b760092f885f8fcfef7ee73afbc7b Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 17:17:11 +0100 Subject: [PATCH 13/16] Enrich the containers test tutorial --- docs/tutorial_advanced.rst | 30 ++++++++++++++----- docs/tutorial_basics.rst | 4 +-- .../advanced/containers/container_test.py | 11 +++---- tutorials/config/settings.py | 4 +++ 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index 9e48ab5ab7..da342904d6 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -671,13 +671,14 @@ ReFrame can be used also to test applications that run inside a container. First, we need to enable the container platform support in ReFrame's configuration and, specifically, at the partition configuration level: .. literalinclude:: ../tutorials/config/settings.py - :lines: 38-58 - :emphasize-lines: 15-20 + :lines: 38-62 + :emphasize-lines: 15-24 For each partition, users can define a list of container platforms supported using the :js:attr:`container_platforms` `configuration parameter `__. -In this case, we define the `Singularity `__ platform, for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``singularity`` module, whenever it needs to run with this container platform. +In this case, we define the `Sarus `__ platform for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``sarus`` module, whenever it needs to run with this container platform. +Furthermore, we define the `Singularity `__ platform, for which the ``singularity`` module needs to be loaded. -The following test will use a Singularity container to run: +The following parameterized test, will create two test cases, one for each of the supported contaiter platforms: .. code-block:: console @@ -686,25 +687,38 @@ The following test will use a Singularity container to run: .. literalinclude:: ../tutorials/advanced/containers/container_test.py :lines: 6- - :emphasize-lines: 11-15 + :emphasize-lines: 11-16 A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform ` attribute. This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. -In this case, the test will be using `Singularity `__ as a container platform. If such a platform is not `configured `__ for the current system, the test will fail. As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image `. +In the ``Singularity`` test case, we add the ``docker://`` prefix to the image name, in order to instruct ``Singularity`` to pull the image from `DockerHub `__. The default command that the container runs can be overwritten by setting the :attr:`command ` attribute of the container platform. The :attr:`image ` is the only mandatory attribute for container-based checks. It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. -ReFrame will run the container as follows: +ReFrame will run the container according to the given platform as follows: -.. code-block:: console +.. code-block:: bash + # Sarus + sarus run --mount=type=bind,source="/path/to/test/stagedir",destination="/rfm_workdir" ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' + + # Singularity singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt' + +In the ``Sarus`` case, ReFrame will prepend the following command in order to pull the container image before running the container: + +.. code-block:: bash + + sarus pull ubuntu:18.04 + + +This is the default behavior of ReFrame, which can be changed if pulling the image is not desired by setting the :attr:`pull_image ` attribute to :class:`False`. By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index c7bb1edd3f..79439d747d 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -323,7 +323,7 @@ Note that you should *not* edit this configuration file in place. Here is how the new configuration file looks like with the needed additions highlighted: .. literalinclude:: ../tutorials/config/settings.py - :lines: 10-24,76-97,130- + :lines: 10-24,80-101,134- :emphasize-lines: 3-15,31-42 Here we define a system named ``catalina`` that has one partition named ``default``. @@ -807,7 +807,7 @@ Let's extend our configuration file for Piz Daint. .. literalinclude:: ../tutorials/config/settings.py - :lines: 10-45,58-66,73- + :lines: 10-45,62-70,77- :emphasize-lines: 16-48,70-101,114-120 diff --git a/tutorials/advanced/containers/container_test.py b/tutorials/advanced/containers/container_test.py index c440c6c0a4..d47bef17ac 100644 --- a/tutorials/advanced/containers/container_test.py +++ b/tutorials/advanced/containers/container_test.py @@ -7,14 +7,15 @@ import reframe.utility.sanity as sn -@rfm.simple_test +@rfm.parameterized_test(['Sarus'], ['Singularity']) class ContainerTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.descr = 'Run commands inside a container' + def __init__(self, platform): + self.descr = f'Run commands inside a container using {platform}' self.valid_systems = ['daint:gpu'] self.valid_prog_environs = ['builtin'] - self.container_platform = 'Singularity' - self.container_platform.image = 'docker://ubuntu:18.04' + image_prefix = 'docker://' if platform == 'Singularity' else '' + self.container_platform = platform + self.container_platform.image = f'{image_prefix}ubuntu:18.04' self.container_platform.command = ( "bash -c 'cat /etc/os-release | tee /rfm_workdir/release.txt'" ) diff --git a/tutorials/config/settings.py b/tutorials/config/settings.py index 1e415d7361..1a2748d990 100644 --- a/tutorials/config/settings.py +++ b/tutorials/config/settings.py @@ -50,6 +50,10 @@ } ], 'container_platforms': [ + { + 'type': 'Sarus', + 'modules': ['sarus'] + }, { 'type': 'Singularity', 'modules': ['singularity'] From be72092fd5659c7d7b82f3636270c36b702c2753 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 25 Feb 2021 17:34:01 +0100 Subject: [PATCH 14/16] Minor refactoring of the unit tests for the deprecated features --- unittests/test_containers.py | 45 ++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 20 deletions(-) diff --git a/unittests/test_containers.py b/unittests/test_containers.py index fed3f0f5f3..79acc9c926 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -191,61 +191,66 @@ def test_run_opts(container_platform, expected_cmd_run_opts): assert container_platform.launch_command() == expected_cmd_run_opts +# Everything from this point is testing deprecated behavior + @pytest.fixture(params=['Docker', 'Singularity', 'Sarus', 'Shifter']) -def container_variant_deprecated(request): +def container_variant_noopt(request): return request.param @pytest.fixture -def platform_deprecated(container_variant_deprecated): - ret = containers.__dict__[container_variant_deprecated]() +def container_platform_noopt(container_variant_noopt): + ret = containers.__dict__[container_variant_noopt]() ret.image = 'image:tag' ret.options = ['--foo'] return ret @pytest.fixture -def expected_run_commands(container_variant_deprecated): - if container_variant_deprecated == 'Docker': +def expected_run_with_commands(container_variant_noopt): + if container_variant_noopt == 'Docker': return ("docker run --rm --foo image:tag bash -c 'cd /rfm_workdir; " "cmd1; cmd2'") - elif container_variant_deprecated == 'Sarus': + elif container_variant_noopt == 'Sarus': return ("sarus run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " "cmd2'") - elif container_variant_deprecated == 'Shifter': + elif container_variant_noopt == 'Shifter': return ("shifter run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " "cmd2'") - elif container_variant_deprecated == 'Singularity': + elif container_variant_noopt == 'Singularity': return ("singularity exec --foo image:tag bash -c 'cd /rfm_workdir; " "cmd1; cmd2'") @pytest.fixture -def expected_run_workdir(container_variant_deprecated): - if container_variant_deprecated == 'Docker': +def expected_run_with_workdir(container_variant_noopt): + if container_variant_noopt == 'Docker': return ("docker run --rm --foo image:tag bash -c 'cd foodir; cmd1; " "cmd2'") - elif container_variant_deprecated == 'Sarus': + elif container_variant_noopt == 'Sarus': return "sarus run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" - elif container_variant_deprecated == 'Shifter': + elif container_variant_noopt == 'Shifter': return "shifter run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" - elif container_variant_deprecated == 'Singularity': + elif container_variant_noopt == 'Singularity': return ("singularity exec --foo image:tag bash -c 'cd foodir; cmd1; " "cmd2'") -def test_run_commands(platform_deprecated, expected_run_commands): +def test_run_with_commands(container_platform_noopt, + expected_run_with_commands): with pytest.warns(warn.ReframeDeprecationWarning): - platform_deprecated.commands = ['cmd1', 'cmd2'] + container_platform_noopt.commands = ['cmd1', 'cmd2'] - assert platform_deprecated.launch_command() == expected_run_commands + found_commands = container_platform_noopt.launch_command() + assert found_commands == expected_run_with_commands -def test_run_workdir(platform_deprecated, expected_run_workdir): +def test_run_with_workdir(container_platform_noopt, expected_run_with_workdir): with pytest.warns(warn.ReframeDeprecationWarning): - platform_deprecated.commands = ['cmd1', 'cmd2'] + container_platform_noopt.commands = ['cmd1', 'cmd2'] with pytest.warns(warn.ReframeDeprecationWarning): - platform_deprecated.workdir = 'foodir' + container_platform_noopt.workdir = 'foodir' - assert platform_deprecated.launch_command() == expected_run_workdir + found_commands = container_platform_noopt.launch_command() + assert found_commands == expected_run_with_workdir From 7a19bb2d7ed66cec65e280c1c2bcf47377cbfb55 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Thu, 25 Feb 2021 18:04:40 +0100 Subject: [PATCH 15/16] Minor tutorial documentation fixes --- docs/tutorial_advanced.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index da342904d6..0cee1a05a5 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -676,9 +676,9 @@ First, we need to enable the container platform support in ReFrame's configurati For each partition, users can define a list of container platforms supported using the :js:attr:`container_platforms` `configuration parameter `__. In this case, we define the `Sarus `__ platform for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``sarus`` module, whenever it needs to run with this container platform. -Furthermore, we define the `Singularity `__ platform, for which the ``singularity`` module needs to be loaded. +Similarly, we add an entry for the `Singularity `__ platform. -The following parameterized test, will create two test cases, one for each of the supported contaiter platforms: +The following parameterized test, will create two tests, one for each of the supported container platforms: .. code-block:: console @@ -694,7 +694,7 @@ This attribute accepts a string that corresponds to the name of the container pl If such a platform is not `configured `__ for the current system, the test will fail. As soon as the container platform to be used is defined, you need to specify the container image to use by setting the :attr:`image `. -In the ``Singularity`` test case, we add the ``docker://`` prefix to the image name, in order to instruct ``Singularity`` to pull the image from `DockerHub `__. +In the ``Singularity`` test variant, we add the ``docker://`` prefix to the image name, in order to instruct ``Singularity`` to pull the image from `DockerHub `__. The default command that the container runs can be overwritten by setting the :attr:`command ` attribute of the container platform. The :attr:`image ` is the only mandatory attribute for container-based checks. From d7e780d15c978145bb2d648dddf8189ab636a9d1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 27 Feb 2021 00:10:16 +0100 Subject: [PATCH 16/16] Extend the container platform interface to accept the stagedir --- reframe/core/containers.py | 35 +++++++------ reframe/core/pipeline.py | 6 +-- unittests/test_containers.py | 95 +++++++++++++++++++++++++----------- 3 files changed, 89 insertions(+), 47 deletions(-) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index f951ac51f2..a57d2ba63a 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -113,7 +113,7 @@ def __init__(self): self.pull_image = True @abc.abstractmethod - def emit_prepare_commands(self): + def emit_prepare_commands(self, stagedir): '''Returns commands for preparing this container for running. Such a command could be for pulling the container image from a @@ -125,10 +125,12 @@ def emit_prepare_commands(self): platform backends. :meta private: + + :arg stagedir: The stage directory of the test. ''' @abc.abstractmethod - def launch_command(self): + def launch_command(self, stagedir): '''Returns the command for running :attr:`commands` with this container platform. @@ -137,6 +139,8 @@ def launch_command(self): platforms. :meta private: + + :arg stagedir: The stage directory of the test. ''' def validate(self): @@ -154,12 +158,13 @@ class Docker(ContainerPlatform): '''Container platform backend for running containers with `Docker `__.''' - def emit_prepare_commands(self): + def emit_prepare_commands(self, stagedir): return [f'docker pull {self.image}'] if self.pull_image else [] - def launch_command(self): - super().launch_command() - run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in self.mount_points] + def launch_command(self, stagedir): + super().launch_command(stagedir) + mount_points = self.mount_points + [(stagedir, _STAGEDIR_MOUNT)] + run_opts = [f'-v "{mp[0]}":"{mp[1]}"' for mp in mount_points] run_opts += self.options if self.command: @@ -188,7 +193,7 @@ def __init__(self): self.with_mpi = False self._command = 'sarus' - def emit_prepare_commands(self): + def emit_prepare_commands(self, stagedir): # The format that Sarus uses to call the images is # //:. If an image was loaded # locally from a tar file, the is 'load'. @@ -197,10 +202,11 @@ def emit_prepare_commands(self): else: return [f'{self._command} pull {self.image}'] - def launch_command(self): - super().launch_command() + def launch_command(self, stagedir): + super().launch_command(stagedir) + mount_points = self.mount_points + [(stagedir, _STAGEDIR_MOUNT)] run_opts = [f'--mount=type=bind,source="{mp[0]}",destination="{mp[1]}"' - for mp in self.mount_points] + for mp in mount_points] if self.with_mpi: run_opts.append('--mpi') @@ -241,12 +247,13 @@ def __init__(self): super().__init__() self.with_cuda = False - def emit_prepare_commands(self): + def emit_prepare_commands(self, stagedir): return [] - def launch_command(self): - super().launch_command() - run_opts = [f'-B"{mp[0]}:{mp[1]}"' for mp in self.mount_points] + def launch_command(self, stagedir): + super().launch_command(stagedir) + mount_points = self.mount_points + [(stagedir, _STAGEDIR_MOUNT)] + run_opts = [f'-B"{mp[0]}:{mp[1]}"' for mp in mount_points] if self.with_cuda: run_opts.append('--nv') diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 03191720b4..c9a8946b6d 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -33,7 +33,7 @@ import reframe.utility.udeps as udeps from reframe.core.backends import getlauncher, getscheduler from reframe.core.buildsystems import BuildSystemField -from reframe.core.containers import _STAGEDIR_MOUNT, ContainerPlatformField +from reframe.core.containers import ContainerPlatformField from reframe.core.deferrable import _DeferredExpression from reframe.core.exceptions import (BuildError, DependencyError, PipelineError, SanityError, @@ -138,6 +138,7 @@ class RegressionMixin(metaclass=RegressionTestMeta): .. versionadded:: 3.4.2 ''' + def __getattribute__(self, name): try: return super().__getattribute__(name) @@ -1289,9 +1290,6 @@ def run(self): 'on the current partition: %s' % e) from None self.container_platform.validate() - self.container_platform.mount_points += [ - (self._stagedir, _STAGEDIR_MOUNT) - ] # We replace executable and executable_opts in case of containers self.executable = self.container_platform.launch_command() diff --git a/unittests/test_containers.py b/unittests/test_containers.py index 79acc9c926..846bda0498 100644 --- a/unittests/test_containers.py +++ b/unittests/test_containers.py @@ -49,59 +49,69 @@ def container_platform(container_variant): def expected_cmd_mount_points(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: return ('docker run --rm -v "/path/one":"/one" ' - '-v "/path/two":"/two" image:tag cmd') + '-v "/path/two":"/two" ' + '-v "/foo":"/rfm_workdir" image:tag cmd') elif container_variant == 'Docker+nocommand': return ('docker run --rm -v "/path/one":"/one" ' - '-v "/path/two":"/two" image:tag') + '-v "/path/two":"/two" ' + '-v "/foo":"/rfm_workdir" image:tag') elif container_variant in {'Sarus', 'Sarus+nopull'}: return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'image:tag') elif container_variant == 'Sarus+mpi': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--mpi image:tag cmd') elif container_variant == 'Sarus+load': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'load/library/image:tag cmd') elif container_variant in {'Shifter', 'Shifter+nopull'}: return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'image:tag cmd') elif container_variant == 'Shifter+nocommand': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'image:tag') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--mpi image:tag cmd') elif container_variant == 'Shifter+load': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' '--mount=type=bind,source="/path/two",destination="/two" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' 'load/library/image:tag cmd') elif container_variant in {'Singularity', 'Singularity+nopull'}: return ('singularity exec -B"/path/one:/one" ' - '-B"/path/two:/two" image:tag cmd') + '-B"/path/two:/two" -B"/foo:/rfm_workdir" image:tag cmd') elif container_variant == 'Singularity+cuda': return ('singularity exec -B"/path/one:/one" ' - '-B"/path/two:/two" --nv image:tag cmd') + '-B"/path/two:/two" -B"/foo:/rfm_workdir" --nv image:tag cmd') elif container_variant == 'Singularity+nocommand': return ('singularity run -B"/path/one:/one" ' - '-B"/path/two:/two" image:tag') + '-B"/path/two:/two" -B"/foo:/rfm_workdir" image:tag') @pytest.fixture @@ -120,59 +130,71 @@ def expected_cmd_prepare(container_variant): def expected_cmd_run_opts(container_variant): if container_variant in {'Docker', 'Docker+nopull'}: return ('docker run --rm -v "/path/one":"/one" ' - '--foo --bar image:tag cmd') + '-v "/foo":"/rfm_workdir" --foo --bar image:tag cmd') if container_variant == 'Docker+nocommand': return ('docker run --rm -v "/path/one":"/one" ' - '--foo --bar image:tag') + '-v "/foo":"/rfm_workdir" --foo --bar image:tag') elif container_variant in {'Shifter', 'Shifter+nopull'}: return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar image:tag cmd') elif container_variant == 'Shifter+nocommand': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar image:tag') elif container_variant == 'Shifter+mpi': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--mpi --foo --bar image:tag cmd') elif container_variant == 'Shifter+load': return ('shifter run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar load/library/image:tag cmd') elif container_variant in {'Sarus', 'Sarus+nopull'}: return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar image:tag cmd') elif container_variant == 'Sarus': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar image:tag cmd') elif container_variant == 'Sarus+load': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar load/library/image:tag cmd') elif container_variant == 'Sarus+nocommand': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--foo --bar image:tag') elif container_variant == 'Sarus+mpi': return ('sarus run ' '--mount=type=bind,source="/path/one",destination="/one" ' + '--mount=type=bind,source="/foo",destination="/rfm_workdir" ' '--mpi --foo --bar image:tag cmd') elif container_variant in {'Singularity'}: - return 'singularity exec -B"/path/one:/one" --foo --bar image:tag cmd' + return ('singularity exec -B"/path/one:/one" -B"/foo:/rfm_workdir" ' + '--foo --bar image:tag cmd') elif container_variant == 'Singularity+cuda': - return ('singularity exec -B"/path/one:/one" ' + return ('singularity exec -B"/path/one:/one" -B"/foo:/rfm_workdir" ' '--nv --foo --bar image:tag cmd') elif container_variant == 'Singularity+nocommand': - return 'singularity run -B"/path/one:/one" --foo --bar image:tag' + return ('singularity run -B"/path/one:/one" -B"/foo:/rfm_workdir" ' + '--foo --bar image:tag') def test_mount_points(container_platform, expected_cmd_mount_points): container_platform.mount_points = [('/path/one', '/one'), ('/path/two', '/two')] - assert container_platform.launch_command() == expected_cmd_mount_points + cmd = container_platform.launch_command('/foo') + assert cmd == expected_cmd_mount_points def test_missing_image(container_platform): @@ -182,13 +204,14 @@ def test_missing_image(container_platform): def test_prepare_command(container_platform, expected_cmd_prepare): - assert container_platform.emit_prepare_commands() == expected_cmd_prepare + commands = container_platform.emit_prepare_commands('/foo') + assert commands == expected_cmd_prepare def test_run_opts(container_platform, expected_cmd_run_opts): container_platform.mount_points = [('/path/one', '/one')] container_platform.options = ['--foo', '--bar'] - assert container_platform.launch_command() == expected_cmd_run_opts + assert container_platform.launch_command('/foo') == expected_cmd_run_opts # Everything from this point is testing deprecated behavior @@ -209,31 +232,45 @@ def container_platform_noopt(container_variant_noopt): @pytest.fixture def expected_run_with_commands(container_variant_noopt): if container_variant_noopt == 'Docker': - return ("docker run --rm --foo image:tag bash -c 'cd /rfm_workdir; " - "cmd1; cmd2'") + return ("docker run --rm -v \"/foo\":\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd /rfm_workdir; cmd1; cmd2'") elif container_variant_noopt == 'Sarus': - return ("sarus run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " - "cmd2'") + return ( + "sarus run " + "--mount=type=bind,source=\"/foo\",destination=\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd /rfm_workdir; cmd1; cmd2'" + ) elif container_variant_noopt == 'Shifter': - return ("shifter run --foo image:tag bash -c 'cd /rfm_workdir; cmd1; " - "cmd2'") + return ( + "shifter run " + "--mount=type=bind,source=\"/foo\",destination=\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd /rfm_workdir; cmd1; cmd2'" + ) elif container_variant_noopt == 'Singularity': - return ("singularity exec --foo image:tag bash -c 'cd /rfm_workdir; " - "cmd1; cmd2'") + return ("singularity exec -B\"/foo:/rfm_workdir\" " + "--foo image:tag bash -c 'cd /rfm_workdir; cmd1; cmd2'") @pytest.fixture def expected_run_with_workdir(container_variant_noopt): if container_variant_noopt == 'Docker': - return ("docker run --rm --foo image:tag bash -c 'cd foodir; cmd1; " - "cmd2'") + return ("docker run --rm -v \"/foo\":\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd foodir; cmd1; cmd2'") elif container_variant_noopt == 'Sarus': - return "sarus run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + return ( + "sarus run " + "--mount=type=bind,source=\"/foo\",destination=\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + ) elif container_variant_noopt == 'Shifter': - return "shifter run --foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + return ( + "shifter run " + "--mount=type=bind,source=\"/foo\",destination=\"/rfm_workdir\" " + "--foo image:tag bash -c 'cd foodir; cmd1; cmd2'" + ) elif container_variant_noopt == 'Singularity': - return ("singularity exec --foo image:tag bash -c 'cd foodir; cmd1; " - "cmd2'") + return ("singularity exec -B\"/foo:/rfm_workdir\" --foo image:tag " + "bash -c 'cd foodir; cmd1; cmd2'") def test_run_with_commands(container_platform_noopt, @@ -241,7 +278,7 @@ def test_run_with_commands(container_platform_noopt, with pytest.warns(warn.ReframeDeprecationWarning): container_platform_noopt.commands = ['cmd1', 'cmd2'] - found_commands = container_platform_noopt.launch_command() + found_commands = container_platform_noopt.launch_command('/foo') assert found_commands == expected_run_with_commands @@ -252,5 +289,5 @@ def test_run_with_workdir(container_platform_noopt, expected_run_with_workdir): with pytest.warns(warn.ReframeDeprecationWarning): container_platform_noopt.workdir = 'foodir' - found_commands = container_platform_noopt.launch_command() + found_commands = container_platform_noopt.launch_command('/foo') assert found_commands == expected_run_with_workdir