diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst index c120b42113..06fc1122eb 100644 --- a/docs/tutorial_advanced.rst +++ b/docs/tutorial_advanced.rst @@ -790,3 +790,63 @@ Therefore, the ``release.txt`` file can now be used in the subsequent sanity che For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`. + + +Writing reusable tests +---------------------- + +.. versionadded:: 3.5.0 + +So far, all the examples shown above were tight to a particular system or configuration, which makes reusing these tests in other systems not straightforward. +However, the introduction of the :py:func:`~reframe.core.pipeline.RegressionTest.parameter` and :py:func:`~reframe.core.pipeline.RegressionTest.variable` ReFrame built-ins solves this problem, eliminating the need to specify any of the test variables in the :func:`__init__` method. +Hence, these parameters and variables can be treated as simple class attributes, which allows us to leverage Python's class inheritance and write more modular tests. +For simplicity, we illustrate this concept with the above :class:`ContainerTest` example, where the goal here is to re-write this test as a library that users can simply import from and derive their tests without having to rewrite the bulk of the test. +Also, for illustrative purposes, we parameterize this library test on a few different image tags (the above example just used ``ubuntu:18.04``) and throw the container commands into a separate bash script just to create some source files. +Thus, removing all the system and configuration specific variables, and moving as many assignments as possible into the class body, the system agnostic library test looks as follows: + +.. code-block:: console + + cat tutorials/advanced/library/lib/__init__.py + + +.. literalinclude:: ../tutorials/advanced/library/lib/__init__.py + :lines: 6- + :emphasize-lines: 8-17 + +Note that the class :class:`ContainerBase` is not decorated since it does not specify the required variables ``valid_systems`` and ``valid_prog_environs``, and it declares the ``platform`` parameter without any defined values assigned. +Hence, the user can simply derive from this test and specialize it to use the desired container platforms. +Since the parameters are defined directly in the class body, the user is also free to override or extend any of the other parameters in a derived test. +In this example, we have parametrized the base test to run with the ``ubuntu:18.04`` and ``ubuntu:20.04`` images, but these values from ``dist`` (and also the ``dist_name`` variable) could be modified by the derived class if needed. + +On the other hand, the rest of the test depends on the values from the test parameters, and a parameter is only assigned a specific value after the class has been instantiated. +Thus, the rest of the test is expressed as hooks, without the need to write anything in the :func:`__init__` method. +In fact, writing the test in this way permits having hooks that depend on undefined variables or parameters. +This is the case with the :func:`set_container_platform` hook, which depends on the undefined parameter ``platform``. +Hence, the derived test **must** define all the required parameters and variables; otherwise ReFrame will notice that the test is not well defined and will raise an error accordingly. + +Before moving onwards to the derived test, note that the :class:`ContainerBase` class takes the additional argument ``pin_prefix=True``, which locks the prefix of all derived tests to this base test. +This will allow the retrieval of the sources located in the library by any derived test, regardless of what their containing directory is. + +.. code-block:: console + + cat tutorials/advanced/library/lib/src/get_os_release.sh + + +.. literalinclude:: ../tutorials/advanced/library/lib/src/get_os_release.sh + :lines: 1- + +Now from the user's perspective, the only thing to do is to import the above base test and specify the required variables and parameters. +For consistency with the above example, we set the ``platform`` parameter to use Sarus and Singularity, and we configure the test to run on Piz Daint with the built-in programming environment. +Hence, the above :class:`ContainerTest` is now reduced to the following: + +.. code-block:: console + + cat tutorials/advanced/library/usr/container_test.py + + +.. literalinclude:: ../tutorials/advanced/library/usr/container_test.py + :lines: 6- + +In a similar fashion, any other user could reuse the above :class:`ContainerBase` class and write the test for their own system with a few lines of code. + +*Happy test sharing!* diff --git a/tutorials/advanced/library/lib/__init__.py b/tutorials/advanced/library/lib/__init__.py new file mode 100644 index 0000000000..fad8bab48c --- /dev/null +++ b/tutorials/advanced/library/lib/__init__.py @@ -0,0 +1,50 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +class ContainerBase(rfm.RunOnlyRegressionTest, pin_prefix=True): + '''Test that asserts the ubuntu version of the image.''' + + # Derived tests must override this parameter + platform = parameter() + image_prefix = variable(str, value='') + + # Parametrize the test on two different versions of ubuntu. + dist = parameter(['18.04', '20.04']) + dist_name = variable(dict, value={ + '18.04': 'Bionic Beaver', + '20.04': 'Focal Fossa', + }) + + @rfm.run_after('setup') + def set_description(self): + self.descr = ( + f'Run commands inside a container using ubuntu {self.dist}' + ) + + @rfm.run_before('run') + def set_container_platform(self): + self.container_platform = self.platform + self.container_platform.image = ( + f'{self.image_prefix}ubuntu:{self.dist}' + ) + self.container_platform.command = ( + "bash -c /rfm_workdir/get_os_release.sh" + ) + + @property + def os_release_pattern(self): + name = self.dist_name[self.dist] + return rf'{self.dist}.\d+ LTS \({name}\)' + + @rfm.run_before('sanity') + def set_sanity_patterns(self): + self.sanity_patterns = sn.all([ + sn.assert_found(self.os_release_pattern, 'release.txt'), + sn.assert_found(self.os_release_pattern, self.stdout) + ]) diff --git a/tutorials/advanced/library/lib/src/get_os_release.sh b/tutorials/advanced/library/lib/src/get_os_release.sh new file mode 100755 index 0000000000..9ca7b5c009 --- /dev/null +++ b/tutorials/advanced/library/lib/src/get_os_release.sh @@ -0,0 +1,2 @@ +#!/bin/bash +cat /etc/os-release | tee /rfm_workdir/release.txt diff --git a/tutorials/advanced/library/usr/container_test.py b/tutorials/advanced/library/usr/container_test.py new file mode 100644 index 0000000000..3b415edf25 --- /dev/null +++ b/tutorials/advanced/library/usr/container_test.py @@ -0,0 +1,19 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import tutorials.advanced.library.lib as lib + + +@rfm.simple_test +class ContainerTest(lib.ContainerBase): + platform = parameter(['Sarus', 'Singularity']) + valid_systems = ['daint:gpu'] + valid_prog_environs = ['builtin'] + + @rfm.run_after('setup') + def set_image_prefix(self): + if self.platform == 'Singularity': + self.image_prefix = 'docker://'