From 1958414ac330025629892818412bdeb48a56845c Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 16 Oct 2019 11:04:01 +0200 Subject: [PATCH 1/4] add doc for the container platform --- docs/advanced.rst | 56 +++++++++++++++++++++++++ docs/configure.rst | 12 ++++++ tutorial/advanced/advanced_example10.py | 25 +++++++++++ tutorial/config/settings.py | 10 +++++ 4 files changed, 103 insertions(+) create mode 100644 tutorial/advanced/advanced_example10.py diff --git a/docs/advanced.rst b/docs/advanced.rst index a7b16d260b..62cbbd4ccf 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -160,6 +160,62 @@ For standard regression tests, this happens at the beginning of the compilation Furthermore, in this particular test the :attr:`executable ` consists only of standard Bash shell commands. For this reason, we can set :attr:`sourcesdir ` to ``None`` informing ReFrame that the test does not have any resources. +Testing applications within containers +-------------------------------------- + +ReFrame can test as well applications that run within a container. +Tests with containers can be written as :class:`RunOnlyRegressionTest `. +However, in such tests, ReFrame ignores the attributes :attr:`executable ` and :attr:`executable_opts ` in favor of the container platform object :attr:`container_platform `. +This is an object of type :class:`ContainerPlatform `. + +All details to run the container are set through the container platform. +The container platform is instantiated by assigning to it a string with the name of the platform. +For instance, when a line like the following is on a test: + +.. code:: python + + self.container_platform = 'Singularity' + +the container platform is initialized and Singularity is set as engine to run the container. + +The container platform requires two main attributes to be set: :attr:`image `, to specify the name of an image from a registry, and :attr:`commands `, to provide the list of commands to be run inside the container. For instance, with the following two lines + +.. code-block:: python + + self.container_platform.image = 'docker://ubuntu:18.04' + self.container_platform.commands = ['pwd', 'ls', 'cat /etc/os-release'] + +ReFrame will run the container as + +.. code:: shell + + singularity exec -B"/path/to/src:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' + +This is, the container is launched with the resources directory folder mounted at ``/rfm_reframe`` and before running any command, the current directory is set to that location. +The commands are then run from ``/rfm_reframe`` one after the other. +Once the commands are executed, the container is stopped and ReFrame goes on to the sanity and/or performance checks. + +A full code of a test with containers can be found on ``/tutorial/advanced/advanced_example10.py``: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + +In this test, the sanity check verifies that the current directory is :attr:`workdir `, that the content of the resources directory is there, and that the operating system running on the container is Ubuntu 18.04. + +In this example, the test uses the default mount point for the resources directory, but if needed, a custom location can be specified with the :attr:`workdir ` attribute of the container platform: + +.. code:: python + + self.container_platform.workdir = '/my_workfir' + +Besides the resources directory, additional mount points can be specified through the container platform's attribute :attr:`mount_points `: + +.. code-block:: python + + self.container_platform.mount_points = [('/path/host/directory1', '/path/container/mount_point1'), + ('/path/host/directory2', '/path/container/mount_point2')] + +Here Singularity is used as container platform, but other container engines like Docker and Shifter are also supported in ReFrame. + Implementing a Compile-Only Regression Test ------------------------------------------- diff --git a/docs/configure.rst b/docs/configure.rst index 3db60d4b38..d9783feb41 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -43,6 +43,11 @@ The following example shows a minimal configuration for the `Piz Daint `__ for more information). +* ``container_platforms``: A dictionary with the container platforms supported in the partition and the corresponding modules to load them. + * ``modules``: A list of modules to be loaded before running a regression test on that partition (default ``[]``). * ``variables``: A set of environment variables to be set before running a regression test on that partition (default ``{}``). diff --git a/tutorial/advanced/advanced_example10.py b/tutorial/advanced/advanced_example10.py new file mode 100644 index 0000000000..a46d3dc4d8 --- /dev/null +++ b/tutorial/advanced/advanced_example10.py @@ -0,0 +1,25 @@ +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.required_version('>=2.20-dev2') +@rfm.simple_test +class Example10Test(rfm.RunOnlyRegressionTest): + def __init__(self, **kwargs): + super().__init__() + self.descr = 'Run commands inside a container' + self.valid_systems = ['daint:gpu'] + self.valid_prog_environs = ['PrgEnv-cray'] + self.container_platform = 'Singularity' + self.container_platform.image = 'docker://ubuntu:18.04' + self.container_platform.workdir = '/workdir' + self.container_platform.commands = [ + 'pwd', 'ls', 'cat /etc/os-release'] + self.sanity_patterns = sn.all([ + sn.assert_found(r'^' + self.container_platform.workdir, + self.stdout), + sn.assert_found(r'^advanced_example1.c', self.stdout), + sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', self.stdout), + ]) + self.maintainers = ['put-your-name-here'] + self.tags = {'tutorial'} diff --git a/tutorial/config/settings.py b/tutorial/config/settings.py index 080271aa75..4839f395ad 100644 --- a/tutorial/config/settings.py +++ b/tutorial/config/settings.py @@ -31,6 +31,11 @@ class ReframeSettings: 'access': ['--constraint=gpu'], 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi'], + 'container_platforms': { + 'Singularity': { + 'modules': ['Singularity'] + } + }, 'descr': 'Hybrid nodes (Haswell/P100)', 'max_jobs': 100 }, @@ -41,6 +46,11 @@ class ReframeSettings: 'access': ['--constraint=mc'], 'environs': ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-intel', 'PrgEnv-pgi'], + 'container_platforms': { + 'Singularity': { + 'modules': ['Singularity'] + } + }, 'descr': 'Multicore nodes (Broadwell)', 'max_jobs': 100 } From 970cf8deed5116d36e59e57cb35dca9910859374 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 21 Oct 2019 18:02:49 +0200 Subject: [PATCH 2/4] Enhance tutorial documentation about container support Also add the corresponding APIs to the reference guide. --- docs/advanced.rst | 113 ++++++++++++------------ docs/reference.rst | 16 ++++ reframe/core/containers.py | 9 +- reframe/core/pipeline.py | 8 +- tutorial/advanced/advanced_example10.py | 5 +- 5 files changed, 84 insertions(+), 67 deletions(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index 62cbbd4ccf..8c1ea8be48 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -160,62 +160,6 @@ For standard regression tests, this happens at the beginning of the compilation Furthermore, in this particular test the :attr:`executable ` consists only of standard Bash shell commands. For this reason, we can set :attr:`sourcesdir ` to ``None`` informing ReFrame that the test does not have any resources. -Testing applications within containers --------------------------------------- - -ReFrame can test as well applications that run within a container. -Tests with containers can be written as :class:`RunOnlyRegressionTest `. -However, in such tests, ReFrame ignores the attributes :attr:`executable ` and :attr:`executable_opts ` in favor of the container platform object :attr:`container_platform `. -This is an object of type :class:`ContainerPlatform `. - -All details to run the container are set through the container platform. -The container platform is instantiated by assigning to it a string with the name of the platform. -For instance, when a line like the following is on a test: - -.. code:: python - - self.container_platform = 'Singularity' - -the container platform is initialized and Singularity is set as engine to run the container. - -The container platform requires two main attributes to be set: :attr:`image `, to specify the name of an image from a registry, and :attr:`commands `, to provide the list of commands to be run inside the container. For instance, with the following two lines - -.. code-block:: python - - self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.commands = ['pwd', 'ls', 'cat /etc/os-release'] - -ReFrame will run the container as - -.. code:: shell - - singularity exec -B"/path/to/src:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' - -This is, the container is launched with the resources directory folder mounted at ``/rfm_reframe`` and before running any command, the current directory is set to that location. -The commands are then run from ``/rfm_reframe`` one after the other. -Once the commands are executed, the container is stopped and ReFrame goes on to the sanity and/or performance checks. - -A full code of a test with containers can be found on ``/tutorial/advanced/advanced_example10.py``: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - -In this test, the sanity check verifies that the current directory is :attr:`workdir `, that the content of the resources directory is there, and that the operating system running on the container is Ubuntu 18.04. - -In this example, the test uses the default mount point for the resources directory, but if needed, a custom location can be specified with the :attr:`workdir ` attribute of the container platform: - -.. code:: python - - self.container_platform.workdir = '/my_workfir' - -Besides the resources directory, additional mount points can be specified through the container platform's attribute :attr:`mount_points `: - -.. code-block:: python - - self.container_platform.mount_points = [('/path/host/directory1', '/path/container/mount_point1'), - ('/path/host/directory2', '/path/container/mount_point2')] - -Here Singularity is used as container platform, but other container engines like Docker and Shifter are also supported in ReFrame. - Implementing a Compile-Only Regression Test ------------------------------------------- @@ -504,3 +448,60 @@ Here is how the new deferred attribute is defined: The behavior of the flexible task allocation is controlled by the ``--flex-alloc-tasks`` command line option. See the corresponding `section `__ for more information. + + +Testing containerized applications +---------------------------------- + +.. versionadded:: 2.20 + + +ReFrame can be used also to test applications that run inside a container. +A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. +The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + +A container-based test in ReFrame requires that the :attr:`container_platform ` is set: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 13 + +This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. +In this case, the test will be using `Singularity `__ as a container platform. +If such a platform is not configured for the current system, the test will fail. +For a complete list of supported container platforms, the user is referred to the `configuration documentation `__. + +As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 14-17 + +These two attributes are mandatory for container-based check. +The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. +It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the :class:`RegressionTest ` are ignored in case of container-based tests. + +In the above example, ReFrame will run the container as follows: + +.. code:: shell + + singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' + +By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. +The user commands then are then run from that directory one after the other. +Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and/or performance checks. + +Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 18 + +Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: + +.. code-block:: python + + self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), + ('/path/to/host/dir2', '/path/to/container/mount_point2')] + + +For a complete list of the available attributes of a specific container platform, the reader is referred to `reference guide `__. diff --git a/docs/reference.rst b/docs/reference.rst index df819c451c..81d516e3a1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -137,3 +137,19 @@ It is up to the concrete build system implementations on how to use or not these :members: :exclude-members: BuildSystemField :show-inheritance: + + +Container platforms +------------------- + +.. versionadded:: 2.20 + +ReFrame can run a regression test inside a container. +To achieve that you have to set the :attr:`reframe.core.pipeline.RegressionTest.container_platform` attribute and then set up the container platform (e.g., image to load, commands to execute). +The :class:`reframe.core.ContainerPlatform` abstract base class define the basic interface and a minimal set of attributes that all concrete container platforms must implement. +Concrete container platforms may also define additional fields that are specific to them. + +.. automodule:: reframe.core.containers + :members: + :exclude-members: ContainerPlatformField + :show-inheritance: diff --git a/reframe/core/containers.py b/reframe/core/containers.py index 763629edf9..db405c22d4 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -9,8 +9,7 @@ class ContainerPlatform(abc.ABC): '''The abstract base class of any container platform. Concrete container platforms inherit from this class and must override the - :func:`emit_prepare_commands()` and :func:`launch_command()` abstract - methods. + :func:`emit_prepare_commands` and :func:`launch_command` abstract methods. ''' image = fields.TypedField('image', str, type(None)) @@ -30,7 +29,7 @@ def emit_prepare_commands(self): '''Returns commands that are necessary before running with this container platform. - :raises: `ContainerError` in case of errors. + :raises: :class:`ContainerError` in case of errors. .. note: This method is relevant only to developers of new container @@ -41,7 +40,7 @@ def emit_prepare_commands(self): def launch_command(self): '''Returns the command for running with this container platform. - :raises: `ContainerError` in case of errors. + :raises: :class:`ContainerError` in case of errors. .. note: This method is relevant only to developers of new container @@ -51,7 +50,7 @@ def launch_command(self): def validate(self): '''Validates this container platform. - :raises: `ContainerError` in case of errors. + :raises: :class:`ContainerError` in case of errors. .. note: This method is relevant only to developers of new container diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 24016e76dd..ed06275a9c 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -224,14 +224,14 @@ class RegressionTest(metaclass=RegressionTestMeta): #: The container platform to be used for this test. #: - #: If the `self.container_platform` is defined on the test, both - #: `self.executable` and `self.executable_opts` are ignored. + #: If the :attr:`container_platform` is defined on the test, both + #: :attr:`executable` and :attr:`executable_opts` are ignored. #: #: :type: :class:`str` or - #: :class:`reframe.core.containers.ContainerPlatform`. + #: :class:`reframe.core.containers.ContainerPlatform`. #: :default: :class:`None`. #: - #: .. versionadded:: 2.19 + #: .. versionadded:: 2.20 container_platform = ContainerPlatformField( 'container_platform', type(None)) diff --git a/tutorial/advanced/advanced_example10.py b/tutorial/advanced/advanced_example10.py index a46d3dc4d8..a84ebe1197 100644 --- a/tutorial/advanced/advanced_example10.py +++ b/tutorial/advanced/advanced_example10.py @@ -12,9 +12,10 @@ def __init__(self, **kwargs): self.valid_prog_environs = ['PrgEnv-cray'] self.container_platform = 'Singularity' self.container_platform.image = 'docker://ubuntu:18.04' - self.container_platform.workdir = '/workdir' self.container_platform.commands = [ - 'pwd', 'ls', 'cat /etc/os-release'] + 'pwd', 'ls', 'cat /etc/os-release' + ] + self.container_platform.workdir = '/workdir' self.sanity_patterns = sn.all([ sn.assert_found(r'^' + self.container_platform.workdir, self.stdout), From d7893b46d435c441a49831c723adbab38cd412b1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 21 Oct 2019 23:47:52 +0200 Subject: [PATCH 3/4] Document container_platforms configuration option --- docs/configure.rst | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/configure.rst b/docs/configure.rst index d9783feb41..3ea07e667f 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -160,7 +160,32 @@ The available partition attributes are the following: * ``environs``: A list of environments, with which ReFrame will try to run any regression tests written for this partition (default ``[]``). The environment names must be resolved inside the ``environments`` section of the ``site_configuration`` dictionary (see `Environments Configuration <#environments-configuration>`__ for more information). -* ``container_platforms``: A dictionary with the container platforms supported in the partition and the corresponding modules to load them. +* ``container_platforms``: *[new in 2.20]* A set of key/value pairs specifying the supported container platforms for this partition and how their environment is set up. + Supported platform names are the following (names are case sensitive): + + - ``Docker``: The `Docker `__ container runtime. + - ``Singularity``: The `Singularity `__ container runtime. + - ``Sarus``: The `Sarus `__ container runtime. + + Each configured container runtime is associated optionally with an environment (modules and environment variables) that is providing it. + This environment is specified as a dictionary in the following format: + + .. code:: python + + { + 'modules': ['mod1', 'mod2', ...] + 'variables': {'ENV1': 'VAL1', 'ENV2': 'VAL2', ...} + } + + + If no special environment arrangement is needed for a configured container platform, you can simply specify an empty dictionary as an environment configuration, as it is shown in the following example: + + .. code:: python + + 'container_platforms': { + 'Docker': {} + } + * ``modules``: A list of modules to be loaded before running a regression test on that partition (default ``[]``). From 4b04af4ad2c5c1de4cd079c686cc68f7769c7226 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 22 Oct 2019 18:38:46 +0200 Subject: [PATCH 4/4] Finalize container documentation. - Also remove public documentation of the `BuildSystemField` class. --- reframe/core/buildsystems.py | 6 --- reframe/core/containers.py | 72 +++++++++++++++++++++--------------- reframe/core/pipeline.py | 23 +++++++++--- 3 files changed, 60 insertions(+), 41 deletions(-) diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index ad9eec38ec..8c4f188323 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -644,12 +644,6 @@ def emit_build_commands(self, environ): class BuildSystemField(fields.TypedField): - '''A field representing a build system. - - You may either assign an instance of :class:`BuildSystem` or a string - representing the name of the concrete class of a build system. - ''' - def __init__(self, fieldname, *other_types): super().__init__(fieldname, BuildSystem, *other_types) diff --git a/reframe/core/containers.py b/reframe/core/containers.py index db405c22d4..53f04058d2 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -12,10 +12,36 @@ class ContainerPlatform(abc.ABC): :func:`emit_prepare_commands` and :func:`launch_command` abstract methods. ''' + #: The container image to be used for running the test. + #: + #: :type: :class:`str` or :class:`None` + #: :default: :class:`None` image = fields.TypedField('image', str, type(None)) + + #: The commands to be executed within the container. + #: + #: :type: :class:`list[str]` + #: :default: ``[]`` commands = fields.TypedField('commands', typ.List[str]) + + #: List of mount point pairs for directories to mount inside the container. + #: + #: Each mount point is specified as a tuple of + #: ``(/path/in/host, /path/in/container)``. + #: + #: :type: :class:`list[tuple[str, str]]` + #: :default: ``[]`` mount_points = fields.TypedField('mount_points', typ.List[typ.Tuple[str, str]]) + + #: The working directory of ReFrame inside the container. + #: + #: This is the directory where the test's stage directory is mounted inside + #: the container. This directory is always mounted regardless if + #: :attr:`mount_points` is set or not. + #: + #: :type: :class:`str` + #: :default: ``/rfm_workdir`` workdir = fields.TypedField('workdir', str, type(None)) def __init__(self): @@ -26,36 +52,30 @@ def __init__(self): @abc.abstractmethod def emit_prepare_commands(self): - '''Returns commands that are necessary before running with this - container platform. + '''Returns commands for preparing this container for running. - :raises: :class:`ContainerError` in case of errors. + Such a command could be for pulling the container image from a + repository. .. note: + This method is relevant only to developers of new container - platforms. + platform backends. + ''' @abc.abstractmethod def launch_command(self): - '''Returns the command for running with this container platform. - - :raises: :class:`ContainerError` in case of errors. + '''Returns the command for running :attr:`commands` with this container + platform. .. note: This method is relevant only to developers of new container platforms. + ''' def validate(self): - '''Validates this container platform. - - :raises: :class:`ContainerError` in case of errors. - - .. note: - This method is relevant only to developers of new container - platforms. - ''' if self.image is None: raise ContainerError('no image specified') @@ -64,8 +84,8 @@ def validate(self): class Docker(ContainerPlatform): - '''An implementation of :class:`ContainerPlatform` for running containers - with Docker.''' + '''Container platform backend for running containers with `Docker + `__.''' def emit_prepare_commands(self): return [] @@ -80,8 +100,7 @@ def launch_command(self): class ShifterNG(ContainerPlatform): - '''An implementation of :class:`ContainerPlatform` for running containers - with ShifterNG.''' + '''Container platform backend for running containers with ShifterNG.''' #: Add an option to the launch command to enable MPI support. #: @@ -111,8 +130,7 @@ def launch_command(self): class Sarus(ShifterNG): - '''An implementation of :class:`ContainerPlatform` for running containers - with Sarus.''' + '''Container platform backend for running containers with Sarus.''' def __init__(self): super().__init__() @@ -120,10 +138,10 @@ def __init__(self): class Singularity(ContainerPlatform): - '''An implementation of :class:`ContainerPlatform` for running containers - with Singularity.''' + '''Container platform backend for running containers with `Singularity + `__.''' - #: Add an option to the launch command to enable CUDA support. + #: Enable CUDA support when launching the container. #: #: :type: boolean #: :default: :class:`False` @@ -149,12 +167,6 @@ def launch_command(self): class ContainerPlatformField(fields.TypedField): - '''A field representing a container platforms. - - You may either assign an instance of :class:`ContainerPlatform:` or a - string representing the name of the concrete class of a container platform. - ''' - def __init__(self, fieldname, *other_types): super().__init__(fieldname, ContainerPlatform, *other_types) diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index ed06275a9c..3e39dc8f99 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -222,18 +222,31 @@ class RegressionTest(metaclass=RegressionTestMeta): #: :default: ``[]`` executable_opts = fields.TypedField('executable_opts', typ.List[str]) - #: The container platform to be used for this test. + #: The container platform to be used for launching this test. #: - #: If the :attr:`container_platform` is defined on the test, both - #: :attr:`executable` and :attr:`executable_opts` are ignored. + #: If this field is set, the test will run inside a container using the + #: specified container runtime. Container-specific options must be defined + #: additionally after this field is set: + #: + #: .. code:: python + #: + #: self.container_platform = 'Singularity' + #: self.container_platform.image = 'docker://ubuntu:18.04' + #: self.container_platform.commands = ['cat /etc/os-release'] + #: + #: If this field is set, :attr:`executable` and :attr:`executable_opts` + #: attributes are ignored. The container platform's :attr:`commands` will + #: be used instead. For more information on the container platform support, + #: see the `tutorial `__ + #: and the `reference guide `__. #: #: :type: :class:`str` or #: :class:`reframe.core.containers.ContainerPlatform`. #: :default: :class:`None`. #: #: .. versionadded:: 2.20 - container_platform = ContainerPlatformField( - 'container_platform', type(None)) + container_platform = ContainerPlatformField('container_platform', + type(None)) #: List of shell commands to execute before launching this job. #: