diff --git a/docs/Makefile b/docs/Makefile index 8c8dbe2c4f..538b2af6a1 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -38,7 +38,7 @@ TARGET_DOCS := \ coverage latest: - @make html + @make html man @touch html/.nojekyll clean: diff --git a/docs/_static/img/async-exec-policy.svg b/docs/_static/img/async-exec-policy.svg new file mode 100644 index 0000000000..198a661d3e --- /dev/null +++ b/docs/_static/img/async-exec-policy.svg @@ -0,0 +1,3 @@ + + +
SE
SE
BU
BU
RU
RU
SA
SA
PE
PE
CL
CL
SE
SE
BU
BU
RU
RU
SE
SE
BU
BU
RU
RU
SA
SA
PE
PE
CL
CL
SA
SA
PE
PE
CL
CLViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/_static/img/pipeline.svg b/docs/_static/img/pipeline.svg index f5379728fd..59ef23fa62 100644 --- a/docs/_static/img/pipeline.svg +++ b/docs/_static/img/pipeline.svg @@ -1,2 +1,3 @@ + -
Pick next test
Pick next test
Supports system?
Supports system?
NO
NO
Supports environment?
Supports environment?
YES
YES
NO
NO
Setup test
Setup test
Compile test
Compile test
Run test
Run test
Check sanity
Check sanity
Check performance
Check performance
Cleanup test resources
Cleanup test resources
YES
YES
Start
Start \ No newline at end of file +
Setup
Setup
Build
Build
Run
Run
Sanity
Sanity
Perf.
Perf.
Cleanup
CleanupViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/_static/img/serial-exec-policy.svg b/docs/_static/img/serial-exec-policy.svg new file mode 100644 index 0000000000..1955dfbacc --- /dev/null +++ b/docs/_static/img/serial-exec-policy.svg @@ -0,0 +1,3 @@ + + +
SE
SE
BU
BU
RU
RU
SA
SA
PE
PE
CL
CL
Idling
Idling
SE
SE
BU
BU
RU
RU
SA
SA
PE
PE
CL
CL
Idling
IdlingViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/about.rst b/docs/about.rst deleted file mode 100644 index 95f82e71f3..0000000000 --- a/docs/about.rst +++ /dev/null @@ -1,56 +0,0 @@ -============= -About ReFrame -============= - -What Is ReFrame? ----------------- - -ReFrame is a framework developed by CSCS to facilitate the writing of regression tests that check the sanity of HPC systems. -Its main goal is to allow users to write their own regression tests without having to deal with all the details of setting up the environment for the test, querying the status of their job, managing the output of the job and looking for sanity and/or performance results. -Users should be concerned only about the logical requirements of their tests. -This allows users' regression checks to be maintained and adapted to new systems easily. - -The user describes his test in a simple Python class and the framework takes care of all the details of the low-level interaction with the system. -The framework is structured in such a way that with a basic knowledge of Python and minimal coding a user can write a regression test, which will be able to run out-of-the-box on a variety of systems and programming environments. - -Writing regression tests in a high-level language, such as Python, allows users to take advantage of the language's higher expressiveness and bigger capabilities compared to classical shell scripting, which is the norm in HPC testing. -This could lead to a more manageable code base of regression tests with significantly reduced maintenance costs. - -ReFrame's Goals ---------------- - -When designing the framework we have set three major goals: - -Productivity - The writer of a regression test should focus only on the logical structure and requirements of the test and should not need to deal with any of the low level details of interacting with the system, e.g., how the environment of the test is loaded, how the associated job is created and has its status checked, how the output parsing is performed etc. -Portability - Configuring the framework to support new systems and system configurations should be easy and should not affect the existing tests. - Also, adding support of a new system in a regression test should require minimal adjustments. -Robustness and ease of use - The new framework must be stable enough and easy to use by non-advanced users. - When the system needs to be returned to users outside normal working hours the personnel in charge should be able to run the regression suite and verify the sanity of the system with a minimal involvement. - -Why ReFrame? ------------- - -HPC systems are highly complex systems in all levels of integration; -from the physical infrastructure up to the software stack provided to the users. -A small change in any of these levels could have an impact on the stability or the performance of the system perceived by the end users. -It is of crucial importance, therefore, not only to make sure that the system is in a sane condition after every maintenance before handing it off to users, but also to monitor its performance during production, so that possible problems are detected early enough and the quality of service is not compromised. - -Regression testing can provide a reliable way to ensure the stability and the performance requirements of the system, provided that sufficient tests exist that cover a wide aspect of the system's operations from both the operators' and users' point of view. -However, given the complexity of HPC systems, writing and maintaining regression tests can be a very time consuming task. -A small change in system configuration or deployment may require adapting hundreds of regression tests at the same time. -Similarly, porting a test to a different system may require significant effort if the new system's configuration is substantially different than that of the system that it was originally written for. - -ReFrame was designed to help HPC support teams to easily write tests that - -* monitor the impact of changes to the system that would affect negatively the users, -* monitor system performance, -* monitor system stability and -* guarantee quality of service. - -And also decrease the amount of time and resources required to - -* write and maintain regression tests and -* port regression tests to other HPC systems. diff --git a/docs/advanced.rst b/docs/advanced.rst deleted file mode 100644 index 49fae44ede..0000000000 --- a/docs/advanced.rst +++ /dev/null @@ -1,684 +0,0 @@ -===================================== -Customizing Further a Regression Test -===================================== - -In this section, we are going to show some more elaborate use cases of ReFrame. -Through the use of more advanced examples, we will demonstrate further customization options which modify the default options of the ReFrame pipeline. -The corresponding scripts as well as the source code of the examples discussed here can be found in the directory ``tutorial/advanced``. - -Working with Makefiles ----------------------- - -We have already shown how you can compile a single source file associated with your regression test. -In this example, we show how ReFrame can leverage Makefiles to build executables. - -Compiling a regression test through a Makefile is straightforward with ReFrame. -If the :attr:`sourcepath ` attribute refers to a directory, then ReFrame will automatically invoke ``make`` in that directory. -More specifically, ReFrame first copies the :attr:`sourcesdir` to the stage directory at the beginning of the compilation phase and then constructs the path ``os.path.join('{STAGEDIR}', self.sourcepath)`` to determine the actual compilation path. -If this is a directory, it will invoke ``make`` in it. - -.. note:: - The :attr:`sourcepath ` attribute must be a relative path refering to a subdirectory of :attr:`sourcesdir `, i.e., relative paths starting with ``..`` will be rejected. - -By default, :attr:`sourcepath ` is the empty string and :attr:`sourcesdir ` is set to ``'src/'``. -As a result, by not specifying a :attr:`sourcepath ` at all, ReFrame will eventually compile the files found in the ``src/`` directory. -This is exactly what our first example here does. - -For completeness, here are the contents of ``Makefile`` provided: - -.. literalinclude:: ../tutorial/advanced/src/Makefile - :language: makefile - -The corresponding ``advanced_example1.c`` source file consists of a simple printing of a message, whose content depends on the preprocessor variable ``MESSAGE``: - -.. literalinclude:: ../tutorial/advanced/src/advanced_example1.c - :language: c - -The purpose of the regression test in this case is to set the preprocessor variable ``MESSAGE`` via ``CPPFLAGS`` and then check the standard output for the message ``SUCCESS``, which indicates that the preprocessor flag has been passed and processed correctly by the Makefile. - -The contents of this regression test are the following (``tutorial/advanced/advanced_example1.py``): - -.. literalinclude:: ../tutorial/advanced/advanced_example1.py - -The important bit here is how we set up the build system for this test: - -.. literalinclude:: ../tutorial/advanced/advanced_example1.py - :lines: 18-19 - :dedent: 4 - - -First, we set the build system to :attr:`Make ` and then set the preprocessor flags for the compilation. -ReFrame will invoke ``make`` as follows: - -.. code:: - - make -j 1 CC='cc' CXX='CC' FC='ftn' NVCC='nvcc' CPPFLAGS='-DMESSAGE' - -The compiler variables (``CC``, ``CXX`` etc.) are set based on the corresponding values specified in the `coniguration of the current environment `__. -You may instruct the build system to ignore the default values from the environment by setting the following: - -.. code-block:: python - - self.build_system.flags_from_environ = False - -In this case, ``make`` will be invoked as follows: - -.. code:: - - make -j 1 CPPFLAGS='-DMESSAGE' - -Notice that the ``-j 1`` option is always generated. -You may change the maximum build concurrency as follows: - -.. code-block:: python - - self.build_system.max_concurrency = 4 - -By setting :attr:`max_concurrency ` to :class:`None`, no limit for concurrent parallel jobs will be placed. -This means that ``make -j`` will be used for building. - -Finally, you may also customize the name of the ``Makefile``. -You can achieve that by setting the corresponding variable of the :class:`Make ` build system: - -.. code-block:: python - - self.build_system.makefile = 'Makefile_custom' - - -More details on ReFrame's build systems, you may find `here `__. - - -Retrieving the source code from a Git repository -================================================ - -It might be the case that a regression test needs to clone its source code from a remote repository. -This can be achieved in two ways with ReFrame. -One way is to set the :attr:`sourcesdir` attribute to :class:`None` and explicitly clone or checkout a repository using the :attr:`prebuild_cmd `: - -.. code-block:: python - - self.sourcesdir = None - self.prebuild_cmd = ['git clone https://github.com/me/myrepo .'] - - -By setting :attr:`sourcesdir` to :class:`None`, you are telling ReFrame that you are going to provide the source files in the stage directory. -The working directory of the :attr:`prebuild_cmd` and :attr:`postbuild_cmd` commands will be the stage directory of the test. - - -An alternative way to retrieve specifically a Git repository is to assign its URL directly to the :attr:`sourcesdir` attribute: - -.. code-block:: python - - self.sourcesdir = 'https://github.com/me/myrepo' - -ReFrame will attempt to clone this repository inside the stage directory by executing ``git clone .`` and will then procede with the compilation as described above. - - -.. note:: - ReFrame recognizes only URLs in the :attr:`sourcesdir` attribute and requires passwordless access to the repository. - This means that the SCP-style repository specification will not be accepted. - You will have to specify it as URL using the ``ssh://`` protocol (see `Git documentation page `__). - - -Add a configuration step before compiling the code -================================================== - -It is often the case that a configuration step is needed before compiling a code with ``make``. -To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make"-style build systems. -It supports `CMake-based `__ projects through the :class:`CMake ` build system, as well as `Autotools-based `__ projects through the :class:`Autotools ` build system. - -For other build systems, you can achieve the same effect using the :class:`Make ` build system and the :attr:`prebuild_cmd ` for performing the configuration step. -The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``: - -.. code-block:: python - - self.prebuild_cmd = ['./custom_configure -with-mylib'] - self.build_system = 'Make' - self.build_system.cppflags = ['-DHAVE_FOO'] - self.build_system.flags_from_environ = False - -The generated build script then will have the following lines: - -.. code-block:: bash - - ./custom_configure -with-mylib - make -j 1 CPPFLAGS='-DHAVE_FOO' - - -Implementing a Run-Only Regression Test ---------------------------------------- - -There are cases when it is desirable to perform regression testing for an already built executable. -The following test uses the ``echo`` Bash shell command to print a random integer between specific lower and upper bounds. -Here is the full regression test (``tutorial/advanced/advanced_example2.py``): - -.. literalinclude:: ../tutorial/advanced/advanced_example2.py - -There is nothing special for this test compared to those presented `earlier `__ except that it derives from the :class:`RunOnlyRegressionTest ` and that it does not contain any resources (``self.sourcesdir = None``). -Note that run-only regression tests may also have resources, as for instance a precompiled executable or some input data. The copying of these resources to the stage directory is performed at the beginning of the run phase. -For standard regression tests, this happens at the beginning of the compilation phase, instead. -Furthermore, in this particular test the :attr:`executable ` consists only of standard Bash shell commands. -For this reason, we can set :attr:`sourcesdir ` to ``None`` informing ReFrame that the test does not have any resources. - -Implementing a Compile-Only Regression Test -------------------------------------------- - -ReFrame provides the option to write compile-only tests which consist only of a compilation phase without a specified executable. -This kind of tests must derive from the :class:`CompileOnlyRegressionTest ` class provided by the framework. -The following example (``tutorial/advanced/advanced_example3.py``) reuses the code of our first example in this section and checks that no warnings are issued by the compiler: - -.. literalinclude:: ../tutorial/advanced/advanced_example3.py - -The important thing to note here is that the standard output and standard error of the tests, accessible through the :attr:`stdout ` and :attr:`stderr ` attributes, are now the corresponding those of the compilation command. -So sanity checking can be done in exactly the same way as with a normal test. - -Leveraging Environment Variables --------------------------------- - -We have already demonstrated in the `tutorial `__ that ReFrame allows you to load the required modules for regression tests and also set any needed environment variables. When setting environment variables for your test through the :attr:`variables ` attribute, you can assign them values of other, already defined, environment variables using the standard notation ``$OTHER_VARIABLE`` or ``${OTHER_VARIABLE}``. -The following regression test (``tutorial/advanced/advanced_example4.py``) sets the ``CUDA_HOME`` environment variable to the value of the ``CUDATOOLKIT_HOME`` and then compiles and runs a simple program: - -.. literalinclude:: ../tutorial/advanced/advanced_example4.py - -Before discussing this test in more detail, let's first have a look in the source code and the Makefile of this example: - -.. literalinclude:: ../tutorial/advanced/src/advanced_example4.c - :language: c - -This program is pretty basic, but enough to demonstrate the use of environment variables from ReFrame. -It simply compares the value of the ``CUDA_HOME`` macro with the value of the environment variable ``CUDA_HOME`` at runtime, printing ``SUCCESS`` if they are not empty and match. -The Makefile for this example compiles this source by simply setting ``CUDA_HOME`` to the value of the ``CUDA_HOME`` environment variable: - -.. literalinclude:: ../tutorial/advanced/src/Makefile_example4 - :language: makefile - -Coming back now to the ReFrame regression test, the ``CUDATOOLKIT_HOME`` environment variable is defined by the ``cudatoolkit`` module. -If you try to run the test, you will see that it will succeed, meaning that the ``CUDA_HOME`` variable was set correctly both during the compilation and the runtime. - -When ReFrame `sets up `__ a test, it first loads its required modules and then sets the required environment variables expanding their values. -This has the result that ``CUDA_HOME`` takes the correct value in our example at the compilation time. - -At runtime, ReFrame will generate the following instructions in the shell script associated with this test: - -.. code-block:: bash - - module load cudatoolkit - export CUDA_HOME=$CUDATOOLKIT_HOME - -This ensures that the environment of the test is also set correctly at runtime. - -Finally, as already mentioned `previously <#working-with-makefiles>`__, since the name of the makefile is not one of the standard ones, it must be set explicitly in the build system: - -.. literalinclude:: ../tutorial/advanced/advanced_example4.py - :lines: 21 - :dedent: 8 - -Setting a Time Limit for Regression Tests ------------------------------------------ - -ReFrame gives you the option to limit the execution time of regression tests. -The following example (``tutorial/advanced/advanced_example5.py``) demonstrates how you can achieve this by limiting the execution time of a test that tries to sleep 100 seconds: - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - -The important bit here is the following line that sets the time limit for the test to one minute: - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 17 - :dedent: 8 - -The :attr:`time_limit ` attribute is a string in the form ``dhms)``. -Time limits are implemented for all the scheduler backends. - -The sanity condition for this test verifies that associated job has been canceled due to the time limit (note that this message is SLURM-specific). - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 20-21 - :dedent: 8 - -Applying a sanity function iteratively --------------------------------------- - -It is often the case that a common sanity pattern has to be applied many times. -In this example we will demonstrate how the above situation can be easily tackled using the :mod:`sanity ` functions offered by ReFrame. -Specifically, we would like to execute the following shell script and check that its output is correct: - -.. literalinclude:: ../tutorial/advanced/src/random_numbers.sh - :language: bash - -The above script simply prints 100 random integers between the limits given by the variables ``LOWER`` and ``UPPER``. -In the corresponding regression test we want to check that all the random numbers printed lie between 90 and 100 ensuring that the script executed correctly. -Hence, a common sanity check has to be applied to all the printed random numbers. -In ReFrame this can achieved by the use of :func:`map ` sanity function accepting a function and an iterable as arguments. -Through :func:`map ` the given function will be applied to all the members of the iterable object. -Note that since :func:`map ` is a sanity function, its execution will be deferred. -The contents of the ReFrame regression test contained in ``advanced_example6.py`` are the following: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - -First the random numbers are extracted through the :func:`extractall ` function as follows: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 18-19 - :dedent: 8 - -The ``numbers`` variable is a deferred iterable, which upon evaluation will return all the extracted numbers. -In order to check that the extracted numbers lie within the specified limits, we make use of the :func:`map ` sanity function, which will apply the :func:`assert_bounded ` to all the elements of ``numbers``. -Additionally, our requirement is that all the numbers satisfy the above constraint and we therefore use :func:`all `. - -There is still a small complication that needs to be addressed. -The :func:`all ` function returns :class:`True` for empty iterables, which is not what we want. -So we must ensure that all the numbers are extracted as well. -To achieve this, we make use of :func:`count ` to get the number of elements contained in ``numbers`` combined with :func:`assert_eq ` to check that the number is indeed 100. -Finally, both of the above conditions have to be satisfied for the program execution to be considered successful, hence the use of the :func:`and_ ` function. -Note that the ``and`` operator is not deferrable and will trigger the evaluation of any deferrable argument passed to it. - -The full syntax for the :attr:`sanity_patterns` is the following: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 20-22 - :dedent: 8 - -Customizing the Generated Job Script ------------------------------------- - -It is often the case that you must run some commands before and/or after the parallel launch of your executable. -This can be easily achieved by using the :attr:`pre_run ` and :attr:`post_run ` attributes of :class:`RegressionTest`. - -The following example is a slightly modified version of the previous one. -The lower and upper limits for the random numbers are now set inside a helper shell script in ``scripts/limits.sh`` and we want also to print the word ``FINISHED`` after our executable has finished. -In order to achieve this, we need to source the helper script just before launching the executable and ``echo`` the desired message just after it finishes. -Here is the test file: - -.. literalinclude:: ../tutorial/advanced/advanced_example7.py - -Notice the use of the :attr:`pre_run` and :attr:`post_run` attributes. -These are list of shell commands that are emitted verbatim in the job script. -The generated job script for this example is the following: - -.. code-block:: bash - - #!/bin/bash -l - #SBATCH --job-name="prerun_demo_check_daint_gpu_PrgEnv-gnu" - #SBATCH --time=0:10:0 - #SBATCH --ntasks=1 - #SBATCH --output=prerun_demo_check.out - #SBATCH --error=prerun_demo_check.err - #SBATCH --constraint=gpu - module load daint-gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - source scripts/limits.sh - srun ./random_numbers.sh - echo FINISHED - -ReFrame generates the job shell script using the following pattern: - -.. code-block:: bash - - #!/bin/bash -l - {job_scheduler_preamble} - {test_environment} - {pre_run} - {parallel_launcher} {executable} {executable_opts} - {post_run} - -The ``job_scheduler_preamble`` contains the directives that control the job allocation. -The ``test_environment`` are the necessary commands for setting up the environment of the test. -This is the place where the modules and environment variables specified in :attr:`modules ` and :attr:`variables ` attributes are emitted. -Then the commands specified in :attr:`pre_run` follow, while those specified in the :attr:`post_run` come after the launch of the parallel job. -The parallel launch itself consists of three parts: - -#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, -#. the regression test executable as specified in the :attr:`executable ` attribute and -#. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. - -A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. - - -Working with parameterized tests --------------------------------- - -.. versionadded:: 2.13 - -We have seen already in the `basic tutorial `__ how we could better organize the tests so as to avoid code duplication by using test class hierarchies. -An alternative technique, which could also be used in parallel with the class hierarchies, is to use `parameterized tests`. -The following is a test that takes a ``variant`` parameter, which controls which variant of the code will be used. -Depending on that value, the test is set up differently: - -.. literalinclude:: ../tutorial/advanced/advanced_example8.py - -If you have already gone through the `tutorial `__, this test can be easily understood. -The new bit here is the ``@parameterized_test`` decorator of the ``MatrixVectorTest`` class. -This decorator takes an arbitrary number of arguments, which are either of a sequence type (i.e., list, tuple etc.) or of a mapping type (i.e., dictionary). -Each of the decorator's arguments corresponds to the constructor arguments of the decorated test that will be used to instantiate it. -In the example shown, the test will be instantiated twice, once passing ``variant`` as ``MPI`` and a second time with ``variant`` passed as ``OpenMP``. -The framework will try to generate unique names for the generated tests by stringifying the arguments passed to the test's constructor: - -.. code-block:: none - - Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/advanced/advanced_example8.py -l - Reframe version: 2.15-dev1 - Launched by user: XXX - Launched on host: daint101 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial/advanced/advanced_example8.py' - Stage dir prefix : current/working/dir/reframe/stage/ - Output dir prefix : current/working/dir/reframe/output/ - Logging dir : current/working/dir/reframe/logs - List of matched checks - ====================== - * MatrixVectorTest_MPI (Matrix-vector multiplication test (MPI)) - * MatrixVectorTest_OpenMP (Matrix-vector multiplication test (OpenMP)) - Found 2 check(s). - - -There are a couple of different ways that we could have used the ``@parameterized_test`` decorator. -One is to use dictionaries for specifying the instantiations of our test class. -The dictionaries will be converted to keyword arguments and passed to the constructor of the test class: - -.. code-block:: python - - @rfm.parameterized_test({'variant': 'MPI'}, {'variant': 'OpenMP'}) - - -Another way, which is quite useful if you want to generate lots of different tests at the same time, is to use either `list comprehensions `__ or `generator expressions `__ for specifying the different test instantiations: - -.. code-block:: python - - @rfm.parameterized_test(*([variant] for variant in ['MPI', 'OpenMP'])) - - -.. note:: - In versions of the framework prior to 2.13, this could be achieved by explicitly instantiating your tests inside the ``_get_checks()`` method. - - -.. tip:: - - Combining parameterized tests and test class hierarchies can offer you a very flexible way for generating multiple related tests at once keeping at the same time the maintenance cost low. - We use this technique extensively in our tests. - - -Flexible Regression Tests -------------------------- - -.. versionadded:: 2.15 - -ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks ` attribute is set to ``<=0``. -In ReFrame's terminology, such tests are called `flexible`. -Negative values indicate the minimum number of tasks that is acceptable for this test (a value of ``-4`` indicates a minimum acceptable number of ``4`` tasks). -A zero value indicates the default minimum number of tasks which is equal to :attr:`num_tasks_per_node `. - -By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted from the command-line. -Flexible tests are very useful for diagnostics tests, e.g., tests for checking the health of a whole set nodes. -In this example, we demonstrate this feature through a simple test that runs ``hostname``. -The test will verify that all the nodes print the expected host name: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - -The first thing to notice in this test is that :attr:`num_tasks ` is set to ``0``. -This is a requirement for flexible tests: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 17 - :dedent: 8 - -The sanity function of this test simply counts the host names and verifies that they are as many as expected: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 19-22 - :dedent: 8 - -Notice, however, that the sanity check does not use :attr:`num_tasks` for verification, but rather a different, custom attribute, the ``num_tasks_assigned``. -This happens for two reasons: - - a. At the time the sanity check expression is created, :attr:`num_tasks` is ``0``. - So the actual number of tasks assigned must be a deferred expression as well. - b. When ReFrame will determine and set the number of tasks of the test, it will not set the :attr:`num_tasks` attribute of the :class:`RegressionTest`. - It will only set the corresponding attribute of the associated job instance. - -Here is how the new deferred attribute is defined: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 26-29 - :dedent: 4 - - -The behavior of the flexible task allocation is controlled by the ``--flex-alloc-nodes`` command line option. -See the corresponding `section `__ for more information. - - -Testing containerized applications ----------------------------------- - -.. versionadded:: 2.20 - - -ReFrame can be used also to test applications that run inside a container. -A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. -The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - -A container-based test in ReFrame requires that the :attr:`container_platform ` is set: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 17 - -This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. -In this case, the test will be using `Singularity `__ as a container platform. -If such a platform is not configured for the current system, the test will fail. -For a complete list of supported container platforms, the user is referred to the `configuration documentation `__. - -As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 18-21 - -These two attributes are mandatory for container-based check. -The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. -It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the :class:`RegressionTest ` are ignored in case of container-based tests. - -In the above example, ReFrame will run the container as follows: - -.. code:: shell - - singularity exec -B"/path/to/test/stagedir:/rfm_workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' - -By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. -The user commands then are then run from that directory one after the other. -Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and/or performance checks. - -Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 22 - -Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: - -.. code-block:: python - - self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), - ('/path/to/host/dir2', '/path/to/container/mount_point2')] - - -For a complete list of the available attributes of a specific container platform, the reader is referred to `reference guide `__. - - -Using dependencies in your tests --------------------------------- - -.. versionadded:: 2.21 - -A ReFrame test may define dependencies to other tests. -An example scenario is to test different runtime configurations of a benchmark that you need to compile, or run a scaling analysis of a code. -In such cases, you don't want to rebuild your test for each runtime configuration. -You could have a build test, which all runtime tests would depend on. -This is the approach we take with the following example, that fetches, builds and runs several `OSU benchmarks `__. -We first create a basic compile-only test, that fetches the benchmarks and builds them for the different programming environments: - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - :lines: 92-106 - -There is nothing particular to that test, except perhaps that you can set :attr:`sourcesdir` to ``None`` even for a test that needs to compile something. -In such a case, you should at least provide the commands that fetch the code inside the :attr:`prebuild_cmd` attribute. - -For the next test we need to use the OSU benchmark binaries that we just built, so as to run the MPI ping-pong benchmark. -Here is the relevant part: - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - :lines: 12-44 - -First, since we will have multiple similar benchmarks, we move all the common functionality to the :class:`OSUBenchmarkTestBase` base class. -Again nothing new here; we are going to use two nodes for the benchmark and we set :attr:`sourcesdir` to ``None``, since none of the benchmark tests will use any additional resources. -The new part comes in with the :class:`OSULatencyTest` test in the following line: - - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - :lines: 32 - -Here we tell ReFrame that this test depends on a test named ``OSUBuildTest``. -This test may or may not be defined in the same test file; all ReFrame needs is the test name. -By default, the :func:`depends_on` function will create dependencies between the individual test cases of the :class:`OSULatencyTest` and the :class:`OSUBuildTest`, such that the :class:`OSULatencyTest` using ``PrgEnv-gnu`` will depend on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-gnu``, but not on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-intel``. -This behaviour can be changed, but we will return to this later. -You can create arbitrary test dependency graphs, but they need to be acyclic. -If ReFrame detects cyclic dependencies, it will refuse to execute the set of tests and will issue an error pointing out the cycle. - -A ReFrame test with dependencies will execute, i.e., enter its `setup` stage, only after `all` of its dependencies have succeeded. -If any of its dependencies fails, the current test will be marked as failure as well. - -The next step for the :class:`OSULatencyTest` is to set its executable to point to the binary produced by the :class:`OSUBuildTest`. -This is achieved with the following specially decorated function: - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - :lines: 37-43 - -The ``@require_deps`` decorator will bind the arguments passed to the decorated function to the result of the dependency that each argument names. -In this case, it binds the ``OSUBuildTest`` function argument to the result of a dependency named ``OSUBuildTest``. -In order for the binding to work correctly the function arguments must be named after the target dependencies. -However, referring to a dependency only by the test's name is not enough, since a test might be associated with multiple programming environments. -For this reason, a dependency argument is actually bound to a function that accepts as argument the name of a target programming environment. -If no arguments are passed to that function, as in this example, the current programming environment is implied, such that ``OSUBuildTest()`` is equivalent to ``OSUBuildTest(self.current_environ.name)``. -This call returns the actual test case of the dependency that has been executed. -This allows you to access any attribute from the target test, as we do in this example by accessing the target test's stage directory, which we use to construct the path of the executable. -This concludes the presentation of the :class:`OSULatencyTest` test. The :class:`OSUBandwidthTest` is completely analogous. - -The :class:`OSUAllreduceTest` shown below is similar to the other two, except that it is parameterized. -It is essentially a scalability test that is running the ``osu_allreduce`` executable created by the :class:`OSUBuildTest` for 2, 4, 8 and 16 nodes. - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - :lines: 69-89 - -The full set of OSU example tests is shown below: - -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py - -Notice that the order dependencies are defined in a test file is irrelevant. -In this case, we define :class:`OSUBuildTest` at the end. -ReFrame will make sure to properly sort the tests and execute them. - -Here is the output when running the OSU tests with the asynchronous execution policy: - -.. code-block:: none - - [==========] Running 7 check(s) - [==========] Started on Wed Mar 25 13:51:06 2020 - - [----------] started processing OSUBuildTest (OSU benchmarks build test) - [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-gnu - [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-intel - [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUBuildTest (OSU benchmarks build test) - - [----------] started processing OSULatencyTest (OSU latency test) - [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-gnu - [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-gnu - [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-intel - [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-intel - [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-pgi - [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-pgi - [----------] finished processing OSULatencyTest (OSU latency test) - - [----------] started processing OSUBandwidthTest (OSU bandwidth test) - [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu - [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu - [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-intel - [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-intel - [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi - [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUBandwidthTest (OSU bandwidth test) - - [----------] started processing OSUAllreduceTest_2 (OSU Allreduce test) - [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu - [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu - [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel - [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel - [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi - [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUAllreduceTest_2 (OSU Allreduce test) - - [----------] started processing OSUAllreduceTest_4 (OSU Allreduce test) - [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu - [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu - [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel - [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel - [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi - [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUAllreduceTest_4 (OSU Allreduce test) - - [----------] started processing OSUAllreduceTest_8 (OSU Allreduce test) - [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu - [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu - [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel - [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel - [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi - [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUAllreduceTest_8 (OSU Allreduce test) - - [----------] started processing OSUAllreduceTest_16 (OSU Allreduce test) - [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu - [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu - [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel - [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel - [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi - [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi - [----------] finished processing OSUAllreduceTest_16 (OSU Allreduce test) - - [----------] waiting for spawned checks to finish - [ OK ] ( 1/21) OSUBuildTest on daint:gpu using PrgEnv-pgi - [ OK ] ( 2/21) OSUBuildTest on daint:gpu using PrgEnv-gnu - [ OK ] ( 3/21) OSUBuildTest on daint:gpu using PrgEnv-intel - [ OK ] ( 4/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi - [ OK ] ( 5/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi - [ OK ] ( 6/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi - [ OK ] ( 7/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi - [ OK ] ( 8/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu - [ OK ] ( 9/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu - [ OK ] (10/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu - [ OK ] (11/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel - [ OK ] (12/21) OSULatencyTest on daint:gpu using PrgEnv-pgi - [ OK ] (13/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu - [ OK ] (14/21) OSULatencyTest on daint:gpu using PrgEnv-gnu - [ OK ] (15/21) OSUBandwidthTest on daint:gpu using PrgEnv-pgi - [ OK ] (16/21) OSUBandwidthTest on daint:gpu using PrgEnv-gnu - [ OK ] (17/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel - [ OK ] (18/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel - [ OK ] (19/21) OSULatencyTest on daint:gpu using PrgEnv-intel - [ OK ] (20/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel - [ OK ] (21/21) OSUBandwidthTest on daint:gpu using PrgEnv-intel - [----------] all spawned checks have finished - - [ PASSED ] Ran 21 test case(s) from 7 check(s) (0 failure(s)) - [==========] Finished on Wed Mar 25 14:37:53 2020 - -Before starting running the tests, ReFrame topologically sorts them based on their dependencies and schedules them for running using the selected execution policy. -With the serial execution policy, ReFrame simply executes the tests to completion as they "arrive", since the tests are already topologically sorted. -In the asynchronous execution policy, tests are spawned and not waited for. -If a test's dependencies have not yet completed, it will not start its execution and a ``DEP`` message will be printed to denote this. - -Finally, ReFrame's runtime takes care of properly cleaning up the resources of the tests respecting dependencies. -Normally when an individual test finishes successfully, its stage directory is cleaned up. -However, if other tests are depending on this one, this would be catastrophic, since most probably the dependent tests would need the outcome of this test. -ReFrame fixes that by not cleaning up the stage directory of a test until all its dependent tests have finished successfully. diff --git a/docs/conf.py b/docs/conf.py index 06daac40a2..bdb93fd63b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,7 +72,7 @@ # General information about the project. project = 'ReFrame' copyright = '2016-2020, CSCS/ETH Zurich' -author = 'CSCS' +author = 'ReFrame Project Developers' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -232,9 +232,15 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'reframe', 'ReFrame Documentation', - [author], 1) + ('manpage', 'reframe', + 'The CLI frontend for managing and executing ReFrame tests', + [author], 1), + ('config_reference', 'reframe.settings', 'ReFrame Configuration Manual', + [author], 8) ] +manpages_url = ( + 'http://man7.org/linux/man-pages/man{section}/{page}.{section}.html' +) # -- Options for Texinfo output ------------------------------------------- diff --git a/docs/config_reference.rst b/docs/config_reference.rst index 4ebecd68e5..c28faa346d 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -2,19 +2,32 @@ Configuration Reference ======================= -.. versionadded:: 3.0 - - -ReFrame's behavior can be configured through its configuration file (see `Configuring ReFrame for Your Site `__), environment variables and command-line options. +ReFrame's behavior can be configured through its configuration file (see :doc:`configure`), environment variables and command-line options. An option can be specified via multiple paths (e.g., a configuration file parameter and an environment variable), in which case command-line options precede environment variables, which in turn precede configuration file options. This section provides a complete reference guide of the configuration options of ReFrame that can be set in its configuration file or specified using environment variables. ReFrame's configuration is in JSON syntax. -The full schema describing it can be found in `schemas/config.json `__ file. +The full schema describing it can be found in |schemas/config.json|_ file. Any configuration file given to ReFrame is validated against this schema. -The syntax we use in the following to describe the different configuration object attributes is a valid query string for the `jq `__ JSON command-line processor. -Along the configuration option, the corresponding environment variable and command-line options are listed, if any. +The syntax we use in the following to describe the different configuration object attributes is a valid query string for the |jq|_ command-line processor. + +.. |jq| replace:: :attr:`jq(1)` +.. _jq: https://stedolan.github.io/jq/manual/ +.. |schemas/config.json| replace:: ``schemas/config.json`` +.. _schemas/config.json: https://github.com/eth-cscs/reframe/blob/master/schemas/config.json +.. |access| replace:: :attr:`access` +.. _access: #.systems[].partitions[].access +.. |basedir| replace:: :attr:`basedir` +.. _basedir: #.logging[].handlers[].basedir +.. |datefmt| replace:: :attr:`datefmt` +.. _datefmt: #.logging[].handlers[].datefmt +.. |environments| replace:: :attr:`environments` +.. _environments: #.environments +.. |handler_name| replace:: :attr:`name` +.. _handler_name: #.logging[].handlers[].name +.. |resources| replace:: :attr:`resources` +.. _resources: #.systems[].partitions[].resources Top-level Configuration @@ -126,10 +139,6 @@ System Configuration .. js:attribute:: .systems[].prefix -.. envvar:: RFM_PREFIX - -.. option:: --prefix - :required: No :default: ``"."`` @@ -138,10 +147,6 @@ System Configuration .. js:attribute:: .systems[].stagedir -.. envvar:: RFM_STAGE_DIR - -.. option:: -s DIR | --stage DIR - :required: No :default: ``"${RFM_PREFIX}/stage"`` @@ -151,10 +156,6 @@ System Configuration .. js:attribute:: .systems[].outputdir -.. envvar:: RFM_OUTPUT_DIR - -.. option:: -o DIR | --output DIR - :required: No :default: ``"${RFM_PREFIX}/output"`` @@ -168,7 +169,7 @@ System Configuration :default: ``"."`` Directory prefix where external test resources (e.g., large input files) are stored. - You may reference this prefix from within a regression test by accessing the corresponding `attribute `__ of the current system. + You may reference this prefix from within a regression test by accessing the :attr:`reframe.core.systems.System.resourcesdir` attribute of the current system. .. js:attribute:: .systems[].partitions @@ -231,7 +232,7 @@ System Partition Configuration - ``srunalloc``: Parallel programs will be launched using `Slurm `__'s ``srun`` command, but job allocation options will also be emitted. This can be useful when combined with the ``local`` job scheduler. - ``ssh``: Parallel programs will be launched using SSH. - This launcher uses the partition’s `access `__ property in order to determine the remote host and any additional options to be passed to the SSH client. + This launcher uses the partition’s |access|_ property in order to determine the remote host and any additional options to be passed to the SSH client. The ssh command will be launched in "batch mode," meaning that password-less access to the remote host must be configured. Here is an example configuration for the ssh launcher: @@ -259,7 +260,7 @@ System Partition Configuration :default: ``[]`` A list of environment names that ReFrame will use to run regression tests on this partition. - Each environment must be defined in the `environments `__ section of the configuration and the definition of the environment must be valid for this partition. + Each environment must be defined in the |environments|_ section of the configuration and the definition of the environment must be valid for this partition. .. js:attribute:: .systems[].partitions[].container_platforms @@ -268,7 +269,7 @@ System Partition Configuration :default: ``[]`` A list for `container platform configuration objects <#container-platform-configuration>`__. - This will allow launching regression tests that use `containers `__ on this partition. + This will allow launching regression tests that use containers on this partition. .. js:attribute:: .systems[].partitions[].modules @@ -297,7 +298,7 @@ System Partition Configuration :default: ``1`` The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. - This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. + This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. .. js:attribute:: .systems[].partitions[].resources @@ -305,7 +306,7 @@ System Partition Configuration :required: No :default: ``[]`` - A list of job scheduler `resource specification <#config_reference.html#custom-job-scheduler-resources>`__ objects. + A list of job scheduler `resource specification `__ objects. @@ -428,8 +429,7 @@ ReFrame allows you to define custom scheduler resources for each partition that } .. note:: - - For the ``pbs`` and ``torque`` backends, options accepted in the `access <#.systems[].partitions[].access>`__ and `resources <#.systems[].partitions[].resources>`__ attributes may either refer to actual ``qsub`` options or may be just resources specifications to be passed to the ``-l`` option. + For the ``pbs`` and ``torque`` backends, options accepted in the |access|_ and |resources|_ attributes may either refer to actual ``qsub`` options or may be just resources specifications to be passed to the ``-l`` option. The backend assumes a ``qsub`` option, if the options passed in these attributes start with a ``-``. @@ -592,14 +592,14 @@ You may define different logger objects per system but *not* per partition. :required: Yes - A list of `logging handlers <#logging-handlers>`__ responsible for handling normal framework output. + A list of logging handlers responsible for handling normal framework output. .. js:attribute:: .logging[].handlers_perflog :required: Yes - A list of logging handlers responsible for handling `performance data <#performance-logging-handlers>`__ from tests. + A list of logging handlers responsible for handling performance data from tests. .. js:attribute:: .logging[].target_systems @@ -629,15 +629,15 @@ All logging handlers share the following set of common attributes: There are the following types available: - ``file``: This handler sends log records to file. - See `here <#the-file-handler>`__ for more details. + See `here <#the-file-log-handler>`__ for more details. - ``filelog``: This handler sends performance log records to files. - See `here <#the-filelog-handler>`__ for more details. + See `here <#the-filelog-log-handler>`__ for more details. - ``graylog``: This handler sends performance log records to Graylog. - See `here <#the-graylog-handler>`__ for more details. + See `here <#the-graylog-log-handler>`__ for more details. - ``stream``: This handler sends log records to a file stream. - See `here <#the-stream-handler>`__ for more details. + See `here <#the-stream-log-handler>`__ for more details. - ``syslog``: This handler sends log records to a Syslog facility. - See `here <#the-syslog-handler>`__ for more details. + See `here <#the-syslog-log-handler>`__ for more details. .. js:attribute:: .logging[].handlers[].level @@ -668,7 +668,7 @@ All logging handlers share the following set of common attributes: - ``%(check_jobid)s``: The job or process id of the job or process associated with the currently executing regression test. If a job or process is not yet created, ``-1`` will be printed. - ``%(check_job_completion_time)s``: The completion time of the job spawned by this regression test. - This timestamp will be formatted according to `datefmt <#.logging[].handlers[].datefmt>`__ handler property. + This timestamp will be formatted according to |datefmt|_ handler property. The accuracy of this timestamp depends on the backend scheduler. The ``slurm`` scheduler `backend <#.systems[].partitions[].scheduler>`__ relies on job accounting and returns the actual termination time of the job. The rest of the backends report as completion time the moment when the framework realizes that the spawned job has finished. @@ -687,13 +687,13 @@ All logging handlers share the following set of common attributes: - ``%(check_system)s``: The system where this test is currently executing. - ``%(check_tags)s``: The tags associated with this test. - ``%(check_perf_lower_thres)s``: The lower threshold of the performance difference from the reference value expressed as a fractional value. - See the `reference `__ attribute of regression tests for more details. + See the :attr:`reframe.core.pipeline.RegressionTest.reference` attribute of regression tests for more details. - ``%(check_perf_ref)s``: The reference performance value of a certain performance variable. - ``%(check_perf_unit)s``: The unit of measurement for the measured performance variable. - - ``%(check_perf_upper_thres)s``: The lower threshold of the performance difference from the reference value expressed as a fractional value. - See the `reference `__ attribute of regression tests for more details. + - ``%(check_perf_upper_thres)s``: The upper threshold of the performance difference from the reference value expressed as a fractional value. + See the :attr:`reframe.core.pipeline.RegressionTest.reference` attribute of regression tests for more details. - ``%(check_perf_value)s``: The performance value obtained for a certain performance variable. - - ``%(check_perf_var)s``: The name of the `performance variable `__ being logged. + - ``%(check_perf_var)s``: The name of the `performance variable `__ being logged. - ``%(osuser)s``: The name of the OS user running ReFrame. - ``%(osgroup)s``: The name of the OS group running ReFrame. - ``%(version)s``: The ReFrame version. @@ -701,7 +701,7 @@ All logging handlers share the following set of common attributes: .. js:attribute:: .logging[].handlers[].datefmt -.. js:attribute:: .logging[].handlers_perflog[].datefmt +.. object:: .logging[].handlers_perflog[].datefmt :required: No :default: ``"%FT%T"`` @@ -721,7 +721,7 @@ The additional properties for the ``file`` handler are the following: .. js:attribute:: .logging[].handlers[].name -.. js:attribute:: .logging[].handlers_perflog[].name +.. object:: .logging[].handlers_perflog[].name :required: Yes @@ -730,7 +730,7 @@ The additional properties for the ``file`` handler are the following: .. js:attribute:: .logging[].handlers[].append -.. js:attribute:: .logging[].handlers_perflog[].append +.. object:: .logging[].handlers_perflog[].append :required: No :default: ``false`` @@ -740,14 +740,14 @@ The additional properties for the ``file`` handler are the following: .. js:attribute:: .logging[].handlers[].timestamp -.. js:attribute:: .logging[].handlers_perflog[].timestamp +.. object:: .logging[].handlers_perflog[].timestamp :required: No :default: ``false`` Append a timestamp to this handler's log file. - This property may also accept a date format as described in the `datefmt <#.logging[].handlers[].datefmt>`__ property. - If the handler's `name <#.logging[].handlers[].name>`__ property is set to ``filename.log`` and this property is set to ``true`` or to a specific timestamp format, the resulting log file will be ``filename_.log``. + This property may also accept a date format as described in the |datefmt|_ property. + If the handler's |handler_name|_ property is set to ``filename.log`` and this property is set to ``true`` or to a specific timestamp format, the resulting log file will be ``filename_.log``. --------------------------- @@ -760,11 +760,7 @@ The additional properties for the ``filelog`` handler are the following: .. js:attribute:: .logging[].handlers[].basedir -.. js:attribute:: .logging[].handlers_perflog[].basedir - -.. envvar:: RFM_PERFLOG_DIR - -.. option:: --perflogdir +.. object:: .logging[].handlers_perflog[].basedir :required: No :default: ``"./perflogs"`` @@ -774,11 +770,11 @@ The additional properties for the ``filelog`` handler are the following: .. js:attribute:: .logging[].handlers[].prefix -.. js:attribute:: .logging[].handlers_perflog[].prefix +.. object:: .logging[].handlers_perflog[].prefix :required: Yes - This is a directory prefix (usually dynamic), appended to the `basedir <#.logging[].handlers_perflog[].basedir>`__, where the performance logs of a test will be stored. + This is a directory prefix (usually dynamic), appended to the |basedir|_, where the performance logs of a test will be stored. This attribute accepts any of the check-specific `formatting placeholders <#.logging[].handlers_perflog[].format>`__. This allows to create dynamic paths based on the current system, partition and/or programming environment a test executes with. For example, a value of ``%(check_system)s/%(check_partition)s`` would generate the following structure of performance log files: @@ -797,9 +793,9 @@ The additional properties for the ``filelog`` handler are the following: ... -.. js:attribute:: .logging[].handlers[].append +.. object:: .logging[].handlers[].append -.. js:attribute:: .logging[].handlers_perflog[].append +.. object:: .logging[].handlers_perflog[].append :required: No :default: ``true`` @@ -816,9 +812,7 @@ The additional properties for the ``graylog`` handler are the following: .. js:attribute:: .logging[].handlers[].address -.. js:attribute:: .logging[].handlers_perflog[].address - -.. envvar:: RFM_GRAYLOG_SERVER +.. object:: .logging[].handlers_perflog[].address :required: Yes @@ -827,7 +821,7 @@ The additional properties for the ``graylog`` handler are the following: .. js:attribute:: .logging[].handlers[].extras -.. js:attribute:: .logging[].handlers_perflog[].extras +.. object:: .logging[].handlers_perflog[].extras :required: No :default: ``{}`` @@ -869,9 +863,9 @@ This handler sends log records to a file stream. The additional properties for the ``stream`` handler are the following: -.. js:attribute:: .logging[].handlers[].name +.. object:: .logging[].handlers[].name -.. js:attribute:: .logging[].handlers_perflog[].name +.. object:: .logging[].handlers_perflog[].name :required: No :default: ``"stdout"`` @@ -893,7 +887,7 @@ The additional properties for the ``syslog`` handler are the following: .. js:attribute:: .logging[].handlers[].socktype -.. js:attribute:: .logging[].handlers_perflog[].socktype +.. object:: .logging[].handlers_perflog[].socktype :required: No :default: ``"udp"`` @@ -907,7 +901,7 @@ The additional properties for the ``syslog`` handler are the following: .. js:attribute:: .logging[].handlers[].facility -.. js:attribute:: .logging[].handlers_perflog[].facility +.. object:: .logging[].handlers_perflog[].facility :required: No :default: ``"user"`` @@ -916,9 +910,9 @@ The additional properties for the ``syslog`` handler are the following: The list of supported facilities can be found `here `__. -.. js:attribute:: .logging[].handlers[].address +.. object:: .logging[].handlers[].address -.. js:attribute:: .logging[].handlers_perflog[].address +.. object:: .logging[].handlers_perflog[].address :required: Yes @@ -989,7 +983,7 @@ The options of an execution mode will be passed to ReFrame as if they were speci The command-line options associated with this execution mode. -.. js:attribute:: .schedulers[].target_systems +.. js:attribute:: .modes[].target_systems :required: No :default: ``["*"]`` @@ -1004,10 +998,6 @@ General Configuration .. js:attribute:: .general[].check_search_path -.. envvar:: RFM_CHECK_SEARCH_PATH - -.. option:: -c PATH | --checkpath PATH - :required: No :default: ``["${RFM_INSTALL_PREFIX}/checks/"]`` @@ -1018,10 +1008,6 @@ General Configuration .. js:attribute:: .general[].check_search_recursive -.. envvar:: RFM_CHECK_SEARCH_RECURSIVE - -.. option:: -R | --recursive - :required: No :default: ``false`` @@ -1031,10 +1017,6 @@ General Configuration .. js:attribute:: .general[].colorize -.. envvar:: RFM_COLORIZE - -.. option:: --nocolor - :required: No :default: ``true`` @@ -1044,10 +1026,6 @@ General Configuration .. js:attribute:: .general[].ignore_check_conflicts -.. envvar:: RFM_IGNORE_CHECK_CONFLICTS - -.. option:: --ignore-check-conflicts - :required: No :default: ``false`` @@ -1056,10 +1034,6 @@ General Configuration .. js:attribute:: .general[].keep_stage_files -.. envvar:: RFM_KEEP_STAGE_FILES - -.. option:: --keep-stage-files - :required: No :default: ``false`` @@ -1068,50 +1042,34 @@ General Configuration .. js:attribute:: .general[].module_map_file -.. envvar:: RFM_MODULE_MAP_FILE - -.. option:: --module-mappings - :required: No :default: ``""`` - File containing `module mappings `__. + File containing module mappings. .. js:attribute:: .general[].module_mappings -.. envvar:: RFM_MODULE_MAPPINGS - -.. option:: -M MAPPING | --map-module MAPPING - :required: No :default: ``[]`` - A list of `module mappings `__. + A list of module mappings. If specified through the environment variable, the mappings must be separated by commas. If specified from command line, multiple module mappings are defined by passing the command line option multiple times. .. js:attribute:: .general[].non_default_craype -.. envvar:: RFM_NON_DEFAULT_CRAYPE - -.. option:: --non-default-craype - :required: No :default: ``false`` Test a non-default Cray Programming Environment. This will emit some special instructions in the generated build and job scripts. - For more details, you may refer `here `__. + See also :option:`--non-default-craype` for more details. .. js:attribute:: .general[].purge_environment -.. envvar:: RFM_PURGE_ENVIRONMENT - -.. option:: --purge-env - :required: No :default: ``false`` @@ -1120,10 +1078,6 @@ General Configuration .. js:attribute:: .general[].save_log_files -.. envvar:: RFM_SAVE_LOG_FILES - -.. option:: --save-log-files - :required: No :default: ``false`` @@ -1141,10 +1095,6 @@ General Configuration .. js:attribute:: .general[].timestamp_dirs -.. envvar:: RFM_TIMESTAMP_DIRS - -.. option:: --timestamp [TIMEFMT] - :required: No :default: ``""`` @@ -1155,10 +1105,6 @@ General Configuration .. js:attribute:: .general[].unload_modules -.. envvar:: RFM_UNLOAD_MODULES - -.. option:: -u MOD | --unload-module MOD - :required: No :default: ``[]`` @@ -1169,10 +1115,6 @@ General Configuration .. js:attribute:: .general[].user_modules -.. envvar:: RFM_USER_MODULES - -.. option:: -m MOD | --module MOD - :required: No :default: ``[]`` @@ -1183,33 +1125,9 @@ General Configuration .. js:attribute:: .general[].verbose -.. envvar:: RFM_VERBOSE - -.. option:: -v | --verbose - :required: No :default: 0 Increase the verbosity level of the output. The higher the number, the more verbose the output will be. If specified from the command line, the command line option must be specified multiple times to increase the verbosity level more than once. - - -Additional Environment Variables --------------------------------- - -Here is a list of environment variables that do not have a configuration option counterpart. - - -.. envvar:: RFM_CONFIG_FILE - -.. option:: -C FILE | --config-file FILE - - The path to ReFrame's configuration file. - - -.. envvar:: RFM_SYSTEM - -.. option:: --system NAME - - The name of the system, whose configuration will be loaded. diff --git a/docs/configure.rst b/docs/configure.rst index 055753c65d..2dce364a82 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -8,7 +8,7 @@ Of course, ReFrame is much more powerful than that. This section will guide you through configuring ReFrame for your HPC cluster. We will use as a starting point a simplified configuration for the `Piz Daint `__ supercomputer at CSCS and we will elaborate along the way. -If you started using ReFrame from version 3.0, you can keep on reading this section, otherwise you are advised to have a look first at the `Migrating to ReFrame 3 `__ page. +If you started using ReFrame from version 3.0, you can keep on reading this section, otherwise you are advised to have a look first at the :doc:`migration_2_to_3` page. ReFrame's configuration file can be either a JSON file or Python file storing the site configuration in a JSON-formatted string. @@ -51,7 +51,7 @@ These sections contain other objects which further define in detail the framewor If you are using a Python file to configure ReFrame, this big JSON configuration object is stored in a special variable called ``site_configuration``. We will explore the basic configuration of ReFrame through the following configuration file that permits ReFrame to run on Piz Daint. -For the complete listing and description of all configuration options, you should refer to the `Configuration Reference `__. +For the complete listing and description of all configuration options, you should refer to the :doc:`config_reference`. .. literalinclude:: ../tutorial/config/settings.py :lines: 10- @@ -71,7 +71,7 @@ In our example we define only one system, namely Piz Daint: .. literalinclude:: ../tutorial/config/settings.py :lines: 11-75 -Each system is associated with a set of properties, which in this case are the following (for a complete list of properties, refer to the `configuration reference `__): +Each system is associated with a set of properties, which in this case are the following: * ``name``: The name of the system. This should be an alphanumeric string (dashes ``-`` are allowed) and it will be used to refer to this system in other contexts. @@ -111,7 +111,7 @@ The basic properties of a partition are the following: Each container platform is an object with a name and list of environment modules to load, in order to enable this platform. For a complete list of the supported container platforms, see `here `__. * ``max_jobs``: The maximum number of concurrent regression tests that may be active (i.e., not completed) on this partition. - This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. + This option is relevant only when ReFrame executes with the `asynchronous execution policy `__. -------------------------- @@ -134,7 +134,6 @@ In other systems, you could define a ReFrame environment to wrap a toolchain (MP Each environment is associated with a name. This name will be used to reference this environment in different contexts, as for example in the ``environs`` property of the system partitions. This environment definition is minimal, since the default values for the rest of the properties serve our purpose. -For a complete list of the environment properties, see the `configuration reference `__. An important feature in ReFrame's configuration, is that you can define section objects differently for different systems or system partitions. In the following, for demonstration purposes, we define ``PrgEnv-gnu`` differently for the ``mc`` partition of the ``daint`` system (notice the condensed form of writing this as ``daint:mc``): @@ -181,12 +180,12 @@ These are the following: * ``level``: The cut-off level for messages reaching this handler. Any message with a lower level number will be filtered out. * ``format``: A format string for formatting the emitted log record. - ReFrame uses the format specifiers from `Python Logging `__, but also defines its owns specifiers. + ReFrame uses the format specifiers from `Python Logging `__, but also defines its owns specifiers. * ``datefmt``: A time format string for formatting timestamps. There are two log record fields that are considered timestamps: (a) ``asctime`` and (b) ``check_job_completion_time``. - ReFrame follows the time formatting syntax of Python's `time.strftime() `__ with a small tweak allowing full RFC3339 compliance when formatting time zone differences (see the `configuration reference `__ for more details). + ReFrame follows the time formatting syntax of Python's `time.strftime() `__ with a small tweak allowing full RFC3339 compliance when formatting time zone differences. -We will not go into the details of the individual handlers here. +We will not go into the details of the individual handlers here. In this particular example we use three handlers of two distinct types: 1. A file handler to print debug messages in the ``reframe.log`` file using a more extensive message format that contains a timestamp, the level name etc. @@ -197,7 +196,7 @@ In this particular example we use three handlers of two distinct types: It might initially seem confusing the fact that there are two ``level`` properties: one at the logger level and one at the handler level. Logging in ReFrame works hierarchically. When a message is logged, an log record is created, which contains metadata about the message being logged (log level, timestamp, ReFrame runtime information etc.). -This log record first goes into ReFrame's internal logger, where the record's level is checked against the logger's level (here ``debug``). +This log record first goes into ReFrame's internal logger, where the record's level is checked against the logger's level (here ``debug``). If the log record's level exceeds the log level threshold from the logger, it is forwarded to the logger's handlers. Then each handler filters the log record differently and takes care of formatting the log record's message appropriately. You can view logger's log level as a general cut off. @@ -209,9 +208,6 @@ The performance handler in this example will create a file per test and per syst Notice in the ``format`` property how the message to be logged is structured such that it can be easily parsed from post processing tools. Apart from file logging, ReFrame offers more advanced performance logging capabilities through Syslog and Graylog. -For a complete description of the logging configuration properties and the different handlers, please refer to the `configuration reference `__. -Section `Running ReFrame `__ provides also examples of logging. - ----------------------------- General configuration options @@ -220,7 +216,7 @@ General configuration options General configuration options of the framework go under the ``general`` section of the configuration file. This section is optional. In this case, we define the search path for ReFrame test files to be the ``tutorial/`` subdirectory and we also instruct ReFrame to recursively search for tests there. -There are several more options that can go into this section, but the reader is referred to the `configuration reference `__ for the complete list. +There are several more options that can go into this section, but the reader is referred to the :doc:`config_reference` for the complete list. --------------------------- @@ -231,7 +227,7 @@ There are finally two more optional configuration sections that are not discusse 1. The ``schedulers`` section holds configuration variables specific to the different scheduler backends and 2. the ``modes`` section defines different execution modes for the framework. - Execution modes are discussed in `Running ReFrame `__. + Execution modes are discussed in the :doc:`pipeline` page. @@ -293,7 +289,7 @@ Let's see some concrete examples: ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=systems/0/partitions .. code:: javascript - + [ { "name": "login", diff --git a/docs/deferrables.rst b/docs/deferrables.rst index 57f3828051..7b77030b11 100644 --- a/docs/deferrables.rst +++ b/docs/deferrables.rst @@ -3,14 +3,14 @@ Understanding the Mechanism of Sanity Functions =============================================== This section describes the mechanism behind the sanity functions that are used for the sanity and performance checking. -Generally, writing a new sanity function is as straightforward as decorating a simple Python function with either the :func:`sanity_function ` or the :func:`@reframe.core.deferrable.deferrable ` decorator. +Generally, writing a new sanity function is as straightforward as decorating a simple Python function with the :func:`reframe.utility.sanity.sanity_function` decorator. However, it is important to understand how and when a deferrable function is evaluated, especially if your function takes as arguments the results of other deferrable functions. What Is a Deferrable Function? ------------------------------ A deferrable function is a function whose a evaluation is deferred to a later point in time. -You can define any function as deferrable by adding the :func:`@sanity_funcion ` or the :func:`@deferrable ` decorator before its definition. +You can define any function as deferrable by wrapping it with the :func:`reframe.utility.sanity.sanity_function` decorator before its definition. The example below demonstrates a simple scenario: .. code-block:: python @@ -184,7 +184,7 @@ If you want to defer the execution of such operators, you should use the corresp In summary deferrable functions have the following characteristics: -* You can make any function deferrable by preceding it with the :func:`@sanity_function ` or the :func:`@deferrable ` decorator. +* You can make any function deferrable by wrapping it with the :func:`reframe.utility.sanity.sanity_function` decorator. * When you call a deferrable function, its body is not executed but its arguments are *captured* and an object representing the deferred function is returned. * You can execute the body of a deferrable function at any later point by calling :func:`evaluate ` on the deferred expression object that it has been returned by the call to the deferred function. * Deferred functions can accept other deferred expressions as arguments and may also return a deferred expression. diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 650caae85f..4f137b1589 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -2,24 +2,7 @@ How Test Dependencies Work In ReFrame ===================================== -Before going into details on how ReFrame treats test dependencies, it is important to understand how tests are actually treated and executed by the runtime. -Normally, a ReFrame test will be tried for different programming environments and different partitions within the same ReFrame run. -These are defined in the test's :func:`__init__` method, but it is not this original object that is being executed by the regression test pipeline. -The following figure explains in more detail the process: - -.. figure:: _static/img/reframe-test-cases.svg - :align: center - :alt: How ReFrame loads and schedules tests for execution. - -When ReFrame loads a test from the disk it unconditionally constructs it executing its :func:`__init__` method. -The practical implication of this is that your test will be instantiated even if it will not run on the current system. -After all the tests are loaded, they are filtered based on the current system and any other criteria (such as programming environment, test attributes etc.) specified by the user (see `Filtering of Regression Tests `__ for more details). -After the tests are filtered, ReFrame creates the actual `test cases` to be run. A test case is essentially a tuple consisting of the test, the system partition and the programming environment to try. -The test that goes into a test case is essentially a `clone` of the original test that was instantiated upon loading. -This ensures that the test case's state is not shared and may not be reused in any case. -Finally, the generated test cases are passed to a `runner` that is responsible for scheduling them for execution based on the selected execution policy. - -Dependencies in ReFrame are defined at the test level using the :func:`depends_on` function, but are projected to the test cases space. +Dependencies in ReFrame are defined at the test level using the :func:`depends_on` function, but are projected to the `test cases `__ space. We will see the rules of that projection in a while. The dependency graph construction and the subsequent dependency analysis happen also at the level of the test cases. @@ -118,16 +101,16 @@ Assume, for example, that :class:`T0` and :class:`T1` are defined as follows: import reframe as rfm import reframe.utility.sanity as sn - - + + @rfm.simple_test class T0(rfm.RegressionTest): def __init__(self): self.valid_systems = ['P0'] self.valid_prog_environs = ['E0'] ... - - + + @rfm.simple_test class T1(rfm.RegressionTest): def __init__(self): @@ -156,7 +139,7 @@ The ``(T0, E1)`` test case would simply have no dependent test cases. Resolving dependencies ---------------------- -As shown in the `tutorial `__, test dependencies would be of limited usage if you were not able to use the results or information of the target tests. +As shown in the :doc:`tutorial_deps`, test dependencies would be of limited usage if you were not able to use the results or information of the target tests. Let's reiterate over the :func:`set_executable` function of the :class:`OSULatencyTest` that we presented previously: .. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py diff --git a/docs/index.rst b/docs/index.rst index e0f7ed6081..1214e2af2b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,8 +2,8 @@ Welcome to ReFrame ================== -ReFrame is a new framework for writing regression tests for HPC systems. -The goal of this framework is to abstract away the complexity of the interactions with the system, separating the logic of a regression test from the low-level details, which pertain to the system configuration and setup. +ReFrame is a high-level framework for writing regression tests for HPC systems. +The goal of the framework is to abstract away the complexity of the interactions with the system, separating the logic of a regression test from the low-level details, which pertain to the system configuration and setup. This allows users to write easily portable regression tests, focusing only on the functionality. Regression tests in ReFrame are simple Python classes that specify the basic parameters of the test. @@ -15,18 +15,17 @@ ReFrame also offers a high-level and flexible abstraction for writing sanity and Writing system regression tests in a high-level modern programming language, like Python, poses a great advantage in organizing and maintaining the tests. Users can create their own test hierarchies or test factories for generating multiple tests at the same time and they can also customize them in a simple and expressive way. -Use Cases -========= +Finally, ReFrame offers a powerful and efficient runtime for running and managing the execution of tests, as well as integration with common logging facilities, where ReFrame can send live data from currently running performance tests. -The ReFrame framework has been in production at `CSCS `__ since the upgrade of the `Piz Daint `__ system in early December 2016. -`Read the full story `__... +Use Cases +========= -Latest Release -============== +A pre-release of ReFrame has been in production at the `Swiss National Supercomputing Centre `__ since early December 2016. +The `first `__ public release was in May 2017 and it is being actively developed since then. +Several HPC centers around the globe have adopted ReFrame for testing and benchmarking their systems in an easy, consistent and reproducible way. +You can read a couple of use cases `here `__. -ReFrame is being actively developed at `CSCS `__. -You can always find the latest release `here `__. Publications ============ @@ -38,7 +37,7 @@ Publications * Slides [`pdf `__] & `Talk `__ @ `FOSDEM'19 `__. * Slides [`pdf `__] @ `4th EasyBuild User Meeting `__. * Slides [`pdf `__] @ `HUST 2018 `__, SC'18. -* Slides [`pdf `__] [`pptx `__] @ `CSCS User Lab Day 2018 `__. +* Slides [`pdf `__] @ `CSCS User Lab Day 2018 `__. * Slides [`pdf `__] @ `HPC Advisory Council 2018 `__. * Slides [`pdf `__] @ `SC17 `__. * Slides [`pdf `__] @ `CUG 2017 `__. @@ -47,19 +46,12 @@ Publications .. toctree:: :caption: Table of Contents - :hidden: - - Getting Started - Configuring ReFrame For Your Site - The Regression Test Pipeline - ReFrame Tutorial - Customizing Further A Regression Test - How Test Dependencies Work in ReFrame - Understanding The Mechanism Of Sanity Functions - Running ReFrame - Use cases - Migrating to ReFrame 3 - About ReFrame - Reference Guide - Sanity Functions Reference - Configuration Reference + :maxdepth: 2 + + started + configure + tutorials + topics + usecases + migration_2_to_3 + manuals diff --git a/docs/manpage.rst b/docs/manpage.rst new file mode 100644 index 0000000000..aae6c41378 --- /dev/null +++ b/docs/manpage.rst @@ -0,0 +1,851 @@ +============================== +ReFrame Command Line Reference +============================== + + +Synopsis +-------- + +.. option:: reframe [OPTION]... ACTION + + +Description +----------- + +ReFrame provides both a `programming interface `__ for writing regression tests and a command-line interface for managing and running the tests, which is detailed here. +The ``reframe`` command is part of ReFrame's frontend. +This frontend is responsible for loading and running regression tests written in ReFrame. +ReFrame executes tests by sending them down to a well defined pipeline. +The implementation of the different stages of this pipeline is part of ReFrame's core architecture, but the frontend is responsible for driving this pipeline and executing tests through it. +There are three basic phases that the frontend goes through, which are described briefly in the following. + + +------------------------------- +Test discovery and test loading +------------------------------- + +This is the very first phase of the frontend. +ReFrame will search for tests in its *check search path* and will load them. +When ReFrame loads a test, it actually *instantiates* it, meaning that it will call its :func:`__init__` method unconditionally whether this test is meant to run on the selected system or not. +This is something that writers of regression tests should bear in mind. + +.. option:: -c, --checkpath=PATH + + A filesystem path where ReFrame should search for tests. + ``PATH`` can be a directory or a single test file. + If it is a directory, ReFrame will search for test files inside this directory load all tests found in them. + This option can be specified multiple times, in which case each ``PATH`` will be searched in order. + + The check search path can also be set using the :envvar:`RFM_CHECK_SEARCH_PATH` environment variable or the :js:attr:`check_search_path` general configuration parameter. + +.. option:: -R, --recursive + + Search for test files recursively in directories found in the check search path. + + This option can also be set using the :envvar:`RFM_CHECK_SEARCH_RECURSIVE` environment variable or the :js:attr:`check_search_recursive` general configuration parameter. + +.. option:: --ignore-check-conflicts + + Ignore tests with conflicting names when loading. + ReFrame requires test names to be unique. + Test names are used as components of the stage and output directory prefixes of tests, as well as for referencing target test dependencies. + This option should generally be avoided unless there is a specific reason. + + This option can also be set using the :envvar:`RFM_IGNORE_CHECK_CONFLICTS` environment variable or the :js:attr:`ignore_check_conflicts` general configuration parameter. + + +-------------- +Test filtering +-------------- + +After all tests in the search path have been loaded, they are first filtered by the selected system. +Any test that is not valid for the current system, it will be filtered out. +The current system is either auto-selected or explicitly specified with the :option:`--system` option. +Tests can be filtered by different attributes and there are specific command line options for achieving this. + + +.. option:: -t, --tag=TAG + + Filter tests by tag. + ``TAG`` is interpreted as a `Python Regular Expression `__; all tests that have at least a matching tag will be selected. + ``TAG`` being a regular expression has the implication that ``-t 'foo'`` will select also tests that define ``'foobar'`` as a tag. + To restrict the selection to tests defining only ``'foo'``, you should use ``-t 'foo$'``. + + This option may be specified multiple times, in which case only tests defining or matching *all* tags will be selected. + +.. option:: -n, --name=NAME + + Filter tests by name. + ``NAME`` is interpreted as a `Python Regular Expression `__; + any test whose name matches ``NAME`` will be selected. + + This option may be specified multiple times, in which case tests with *any* of the specified names will be selected: + ``-n NAME1 -n NAME2`` is therefore equivalent to ``-n 'NAME1|NAME2'``. + +.. option:: -x, --exclude=NAME + + Exclude tests by name. + ``NAME`` is interpreted as a `Python Regular Expression `__; + any test whose name matches ``NAME`` will be excluded. + + This option may be specified multiple times, in which case tests with *any* of the specified names will be excluded: + ``-x NAME1 -x NAME2`` is therefore equivalent to ``-x 'NAME1|NAME2'``. + +.. option:: -p, --prgenv=NAME + + Filter tests by programming environment. + ``NAME`` is interpreted as a `Python Regular Expression `__; + any test for which at least one valid programming environment is matching ``NAME`` will be selected. + + This option may be specified multiple times, in which case only tests matching all of the specified programming environments will be selected. + +.. option:: --gpu-only + + Select tests that can run on GPUs. + These are all tests with :attr:`num_gpus_per_node` greater than zero. + This option and :option:`--cpu-only` are mutually exclusive. + +.. option:: --cpu-only + + Select tests that do not target GPUs. + These are all tests with :attr:`num_gpus_per_node` equals to zero + This option and :option:`--gpu-only` are mutually exclusive. + + The :option:`--gpu-only` and :option:`--cpu-only` check only the value of the :attr:`num_gpus_per_node` attribute of tests. + The value of this attribute is not required to be non-zero for GPU tests. + Tests may or may not make use of it. + + +.. option:: --skip-system-check + + Do not filter tests against the selected system. + + +.. option:: --skip-prgenv-check + + Do not filter tests against programming environments. + Even if the :option:`-p` option is not specified, ReFrame will filter tests based on the programming environments defined for the currently selected system. + This option disables that filter completely. + + +------------ +Test actions +------------ + +ReFrame will finally act upon the selected tests. +There are currently two actions that can be performed on tests: (a) list the tests and (b) execute the tests. +An action must always be specified. + + +.. option:: -l, --list + + List selected tests. + A single line per test is printed. + + +.. option:: -L, --list-detailed + + List selected tests providing detailed information per test. + + +.. option:: -r, --run + + Execute the selected tests. + + +If more than one action options are specified, :option:`-l` precedes :option:`-L`, which in turn precedes :option:`-r`. + + +---------------------------------- +Options controlling ReFrame output +---------------------------------- + +.. option:: --prefix=DIR + + General directory prefix for ReFrame-generated directories. + The base stage and output directories (see below) will be specified relative to this prefix if not specified explicitly. + + This option can also be set using the :envvar:`RFM_PREFIX` environment variable or the :js:attr:`prefix` system configuration parameter. + +.. option:: -o, --output=DIR + + Directory prefix for test output files. + When a test finishes successfully, ReFrame copies important output files to a test-specific directory for future reference. + This test-specific directory is of the form ``{output_prefix}/{system}/{partition}/{environment}/{test_name}``, + where ``output_prefix`` is set by this option. + The test files saved in this directory are the following: + + - The ReFrame-generated build script, if not a run-only test. + - The standard output and standard error of the build phase, if not a run-only test. + - The ReFrame-generated job script, if not a compile-only test. + - The standard output and standard error of the run phase, if not a compile-only test. + - Any additional files specified by the :attr:`keep_files` regression test attribute. + + This option can also be set using the :envvar:`RFM_OUTPUT_DIR` environment variable or the :js:attr:`outputdir` system configuration parameter. + + +.. option:: -s, --stage=DIR + + Directory prefix for staging test resources. + ReFrame does not execute tests from their original source directory. + Instead it creates a test-specific stage directory and copies all test resources there. + It then changes to that directory and executes the test. + This test-specific directory is of the form ``{stage_prefix}/{system}/{partition}/{environment}/{test_name}``, + where ``stage_prefix`` is set by this option. + If a test finishes successfully, its stage directory will be removed. + + This option can also be set using the :envvar:`RFM_STAGE_DIR` environment variable or the :js:attr:`stagedir` system configuration parameter. + +.. option:: --timestamp[=TIMEFMT] + + Append a timestamp to the output and stage directory prefixes. + ``TIMEFMT`` can be any valid :manpage:`strftime(3)` time format. + If not specified, ``TIMEFMT`` is set to ``%FT%T``. + + This option can also be set using the :envvar:`RFM_TIMESTAMP_DIRS` environment variable or the :js:attr:`timestamp_dirs` general configuration parameter. + + +.. option:: --perflogdir=DIR + + Directory prefix for logging performance data. + This option is relevant only to the ``filelog`` `logging handler `__. + + This option can also be set using the :envvar:`RFM_PERFLOG_DIR` environment variable or the :js:attr:`basedir` logging handler configuration parameter. + + +.. option:: --keep-stage-files + + Keep test stage directories even for tests that finish successfully. + + This option can also be set using the :envvar:`RFM_KEEP_STAGE_FILES` environment variable or the :js:attr:`keep_stage_files` general configuration parameter. + +.. option:: --save-log-files + + Save ReFrame log files in the output directory before exiting. + Only log files generated by ``file`` `log handlers `__ will be copied. + + + This option can also be set using the :envvar:`RFM_SAVE_LOG_FILES` environment variable or the :js:attr:`save_log_files` general configuration parameter. + + +------------------------------------- +Options controlling ReFrame execution +------------------------------------- + +.. option:: --force-local + + Force local execution of tests. + Execute tests as if all partitions of the currently selected system had a ``local`` scheduler. + +.. option:: --skip-sanity-check + + Skip sanity checking phase. + + +.. option:: --skip-performance-check + + Skip performance checking phase. + The phase is completely skipped, meaning that performance data will *not* be logged. + +.. option:: --strict + + Enforce strict performance checking, even if a performance test is marked as not performance critical by having set its :attr:`strict_check` attribute to :class:`False`. + + +.. option:: --exec-policy=POLICY + + The execution policy to be used for running tests. + There are two policies defined: + + - ``serial``: Tests will be executed sequentially. + - ``async``: Tests will be executed asynchronously. + This is the default policy. + + The ``async`` execution policy executes the run phase of tests asynchronously by submitting their associated jobs in a non-blocking way. + ReFrame's runtime monitors the progress of each test and will resume the pipeline execution of an asynchronously spawned test as soon as its run phase has finished. + Note that the rest of the pipeline stages are still executed sequentially in this policy. + + Concurrency can be controlled by setting the :js:attr:`max_jobs` system partition configuration parameter. + As soon as the concurrency limit is reached, ReFrame will first poll the status of all its pending tests to check if any execution slots have been freed up. + If there are tests that have finished their run phase, ReFrame will keep pushing tests for execution until the concurrency limit is reached again. + If no execution slots are available, ReFrame will throttle job submission. + + +.. option:: --mode=MODE + + ReFrame execution mode to use. + An execution mode is simply a predefined invocation of ReFrame that is set with the :js:attr:`modes` configuration parameter. + If an option is specified both in an execution mode and in the command-line, then command-line takes precedence. + +.. option:: --max-retries=NUM + + The maximum number of times a failing test can be retried. + The test stage and output directories will receive a ``_retry`` suffix every time the test is retried. + + +---------------------------------- +Options controlling job submission +---------------------------------- + +.. option:: -A, --account=NAME + + Submit test-related jobs using the account ``NAME``. + This option is relevant only for the Slurm backend and translates to Slurm's ``--account`` option and it precedes any options specified in the :js:attr:`access` system partition configuration parameter. + + +.. option:: -P, --partition=NAME + + Submit test-related jobs using scheduler partition ``NAME``. + This option is relevant only for the Slurm, PBS and Torque backends and it translates to the ``--partition`` or ``-q`` scheduler options, respectively and it precedes any options specified in the :js:attr:`access` system partition configuration parameter. + +.. option:: --reservation=NAME + + Submit test-related jobs on reservation ``NAME``. + This option is relevant only for the Slurm backend and translates to Slurm's ``--reservation`` option and it precedes any options specified in the :js:attr:`access` system partition configuration parameter. + +.. option:: --nodelist=NODES + + Submit test-related jobs on the selected nodes. + This option is relevant only for the Slurm backend and translates to Slurm's ``--nodelist`` option and it precedes any options specified in the :js:attr:`access` system partition configuration parameter. + The same node range naming conventions as of Slurm apply. + + +.. option:: --exclude-nodes=NODES + + Do not submit test-related jobs on the selected nodes. + This option is relevant only for the Slurm backend and translates to Slurm's ``--exclude`` option and it precedes any options specified in the :js:attr:`access` system partition configuration parameter. + The same node range naming conventions as of Slurm apply. + +.. option:: --job-option=OPTION + + Pass ``OPTION`` directly to the job scheduler backend. + This option will be emitted after any options specified in the :js:attr:`access` system partition configuration parameter. + + +------------------------ +Flexible node allocation +------------------------ + +ReFrame can automatically set the number of tasks of a test, if its :attr:`num_tasks ` attribute is set to a value less than or equal to zero. +This scheme is conveniently called *flexible node allocation* and is valid only for the Slurm backend. +When allocating nodes automatically, ReFrame will take into account all node limiting factors, such as partition :js:attr:`access` options, and any job submission control options described above. +Nodes from this pool are allocated according to different policies. +If no node can be selected, the test will be marked as a failure with an appropriate message. + +.. option:: --flex-alloc-nodes[=POLICY] + + Set the flexible node allocation policy. + Available values are the following: + + - ``all``: Flexible tests will be assigned as many tasks as needed in order to span over *all* the nodes of the node pool. + - ``idle``: Flexible tests will be assigned as many tasks as needed in order to span over the *idle* nodes of the node pool. + Querying of the node state and submission of the test job are two separate steps not executed atomically. + It is therefore possible that the number of tasks assigned does not correspond to the actual idle nodes. + + This is the default policy. + - Any positive integer: Flexible tests will be assigned as many tasks as needed in order to span over the specified number of nodes from the node pool. + +.. option:: --flex-alloc-tasks[=POLICY] + + .. deprecated:: 2.21 + + Please use |--flex-alloc-nodes|_ instead. + +.. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` +.. _--flex-alloc-nodes: #cmdoption-flex-alloc-nodes + +--------------------------------------- +Options controlling ReFrame environment +--------------------------------------- + +ReFrame offers the ability to dynamically change its environment as well as the environment of tests. +It does so by leveraging the selected system's environment modules system. + +.. option:: -m, --module=NAME + + Load environment module ``NAME`` before acting on any tests. + This option may be specified multiple times, in which case all specified modules will be loaded in order. + ReFrame will *not* perform any automatic conflict resolution. + + This option can also be set using the :envvar:`RFM_USER_MODULES` environment variable or the :js:attr:`user_modules` general configuration parameter. + + +.. option:: -u, --unload-module=NAME + + Unload environment module ``NAME`` before acting on any tests. + This option may be specified multiple times, in which case all specified modules will be unloaded in order. + + This option can also be set using the :envvar:`RFM_UNLOAD_MODULES` environment variable or the :js:attr:`unload_modules` general configuration parameter. + + +.. option:: --purge-env + + Unload all environment modules before acting on any tests. + This will unload also sticky Lmod modules. + + This option can also be set using the :envvar:`RFM_PURGE_ENVIRONMENT` environment variable or the :js:attr:`purge_environment` general configuration parameter. + + +.. option:: --non-default-craype + + Test a non-default Cray Programming Environment. + Since CDT 19.11, this option can be used in conjunction with :option:`-m`, which will load the target CDT. + For example: + + .. code:: bash + + reframe -m cdt/20.03 --non-default-craype -r + + This option causes ReFrame to properly set the ``LD_LIBRARY_PATH`` for such cases. + It will emit the following code after all the environment modules of a test have been loaded: + + .. code:: bash + + export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH + + This option can also be set using the :envvar:`RFM_NON_DEFAULT_CRAYPE` environment variable or the :js:attr:`non_default_craype` general configuration parameter. + +.. option:: -M, --map-module=MAPPING + + Apply a module mapping. + ReFrame allows manipulating test modules on-the-fly using module mappings. + A module mapping has the form ``old_module: module1 [module2]...`` and will cause ReFrame to replace a module with another list of modules upon load time. + For example, the mapping ``foo: foo/1.2`` will load module ``foo/1.2`` whenever module ``foo`` needs to be loaded. + A mapping may also be self-referring, e.g., ``gnu: gnu gcc/10.1``, however cyclic dependencies in module mappings are not allowed and ReFrame will issue an error if it detects one. + This option is especially useful for running tests using a newer version of a software or library. + + This option may be specified multiple times, in which case multiple mappings will be applied. + + This option can also be set using the :envvar:`RFM_MODULE_MAPPINGS` environment variable or the :js:attr:`module_mappings` general configuration parameter. + + +.. option:: --module-mappings=FILE + + A file containing module mappings. + Each line of the file contains a module mapping in the form described in the :option:`-M` option. + This option may be combined with the :option:`-M` option, in which case module mappings specified will be applied additionally. + + This option can also be set using the :envvar:`RFM_MODULE_MAP_FILE` environment variable or the :js:attr:`module_map_file` general configuration parameter. + + +--------------------- +Miscellaneous options +--------------------- + +.. option:: -C --config-file=FILE + + Use ``FILE`` as configuration file for ReFrame. + + This option can also be set using the :envvar:`RFM_CONFIG_FILE` environment variable. + +.. option:: --show-config[=PARAM] + + Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit. + The parameter value is printed in JSON format. + If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown. + Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter. + The ``/`` character acts as a selector of configuration object properties or an index in array objects. + The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property. + Here are some example queries: + + - Retrieve all the partitions of the current system: + + .. code:: bash + + reframe --show-config=systems/0/partitions + + - Retrieve the job scheduler of the partition named ``default``: + + .. code:: bash + + reframe --show-config=systems/0/partitions/@default/scheduler + + - Retrieve the check search path for system ``foo``: + + .. code:: bash + + reframe --system=foo --show-config=general/0/check_search_path + + +.. option:: --system=NAME + + Load the configuration for system ``NAME``. + The ``NAME`` must be a valid system name in the configuration file. + It may also have the form ``SYSNAME:PARTNAME``, in which case the configuration of system ``SYSNAME`` will be loaded, but as if it had ``PARTNAME`` as its sole partition. + Of course, ``PARTNAME`` must be a valid partition of system ``SYSNAME``. + If this option is not specified, ReFrame will try to pick the correct configuration entry automatically. + It does so by trying to match the hostname of the current machine again the hostname patterns defined in the :js:attr:`hostnames` system configuration parameter. + The system with the first match becomes the current system. + For Cray systems, ReFrame will first look for the *unqualified machine name* in ``/etc/xthostname`` before trying retrieving the hostname of the current machine. + + This option can also be set using the :envvar:`RFM_SYSTEM` environment variable. + +.. option:: --failure-stats + + Print failure statistics at the end of the run. + + +.. option:: --performance-report + + Print a performance report for all the performance tests that have been run. + The report shows the performance values retrieved for the different performance variables defined in the tests. + + +.. option:: --nocolor + + Disable output coloring. + + This option can also be set using the :envvar:`RFM_COLORIZE` environment variable or the :js:attr:`colorize` general configuration parameter. + + +.. option:: -v, --verbose + + Increase verbosity level of output. + This option can be specified multiple times. + Every time this option is specified, the verbosity level will be increased by one. + There are the following message levels in ReFrame listed in increasing verbosity order: + ``critical``, ``error``, ``warning``, ``info``, ``verbose`` and ``debug``. + The base verbosity level of the output is defined by the :js:attr:`level` `stream logging handler `__ configuration parameter. + + This option can also be set using the :envvar:`RFM_VERBOSE` environment variable or the :js:attr:`verbose` general configuration parameter. + + +.. option:: -V, --version + + Print version and exit. + + +.. option:: -h, --help + + Print a short help message and exit. + + +Environment +----------- + +Several aspects of ReFrame can be controlled through environment variables. +Usually environment variables have counterparts in command line options or configuration parameters. +In such cases, command-line options take precedence over environment variables, which in turn precede configuration parameters. +Boolean environment variables can have any value of ``true``, ``yes`` or ``y`` (case insensitive) to denote true and any value of ``false``, ``no`` or ``n`` (case insensitive) to denote false. + +Here is an alphabetical list of the environment variables recognized by ReFrame: + + +.. envvar:: RFM_CHECK_SEARCH_PATH + + A colon-separated list of filesystem paths where ReFrame should search for tests. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-c` + Associated configuration parameter :js:attr:`check_search_path` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_CHECK_SEARCH_RECURSIVE + + Search for test files recursively in directories found in the check search path. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-R` + Associated configuration parameter :js:attr:`check_search_recursive` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_COLORIZE + + Enable output coloring. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--nocolor` + Associated configuration parameter :js:attr:`colorize` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_CONFIG_FILE + + Set the configuration file for ReFrame. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-C` + Associated configuration parameter N/A + ================================== ================== + + +.. envvar:: RFM_GRAYLOG_SERVER + + The address of the Graylog server to send performance logs. + The address is specified in ``host:port`` format. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :js:attr:`address` graylog log handler configuration parameter + ================================== ================== + + +.. envvar:: RFM_IGNORE_CHECK_CONFLICTS + + Ignore tests with conflicting names when loading. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--ignore-check-conflicts` + Associated configuration parameter :js:attr:`ignore_check_conflicts` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_KEEP_STAGE_FILES + + Keep test stage directories even for tests that finish successfully. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--keep-stage-files` + Associated configuration parameter :js:attr:`keep_stage_files` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_MODULE_MAP_FILE + + A file containing module mappings. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--module-mappings` + Associated configuration parameter :js:attr:`module_map_file` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_MODULE_MAPPINGS + + A comma-separated list of module mappings. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-M` + Associated configuration parameter :js:attr:`module_mappings` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_NON_DEFAULT_CRAYPE + + Test a non-default Cray Programming Environment. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--non-default-craype` + Associated configuration parameter :js:attr:`non_default_craype` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_OUTPUT_DIR + + Directory prefix for test output files. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-o` + Associated configuration parameter :js:attr:`outputdir` system configuration parameter + ================================== ================== + + +.. envvar:: RFM_PERFLOG_DIR + + Directory prefix for logging performance data. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--perflogdir` + Associated configuration parameter :js:attr:`basedir` logging handler configuration parameter + ================================== ================== + + +.. envvar:: RFM_PREFIX + + General directory prefix for ReFrame-generated directories. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--prefix` + Associated configuration parameter :js:attr:`prefix` system configuration parameter + ================================== ================== + + +.. envvar:: RFM_PURGE_ENVIRONMENT + + Unload all environment modules before acting on any tests. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--purge-env` + Associated configuration parameter :js:attr:`purge_environment` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_SAVE_LOG_FILES + + Save ReFrame log files in the output directory before exiting. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--save-log-files` + Associated configuration parameter :js:attr:`save_log_files` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_STAGE_DIR + + Directory prefix for staging test resources. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-s` + Associated configuration parameter :js:attr:`stagedir` system configuration parameter + ================================== ================== + + +.. envvar:: RFM_SYSTEM + + Set the current system. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--system` + Associated configuration parameter N/A + ================================== ================== + + +.. envvar:: RFM_TIMESTAMP_DIRS + + Append a timestamp to the output and stage directory prefixes. + + .. table:: + :align: left + + ================================== ================== + Associated command line option |--timestamp|_ + Associated configuration parameter :js:attr:`timestamp_dirs` general configuration parameter. + ================================== ================== + +.. |--timestamp| replace:: :attr:`--timestamp` +.. _--timestamp: #cmdoption-timestamp + + + +.. envvar:: RFM_UNLOAD_MODULES + + A comma-separated list of environment modules to be unloaded before acting on any tests. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-u` + Associated configuration parameter :js:attr:`unload_modules` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_USER_MODULES + + A comma-separated list of environment modules to be loaded before acting on any tests. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-m` + Associated configuration parameter :js:attr:`user_modules` general configuration parameter + ================================== ================== + + +.. envvar:: RFM_VERBOSE + + Increase verbosity level of output. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`-v` + Associated configuration parameter :js:attr:`verbose` general configuration parameter + ================================== ================== + + + +Configuration File +------------------ + +The configuration file of ReFrame defines the systems and environments to test as well as parameters controlling its behavior. +Upon start up ReFrame checks for configuration files in the following locations in that order: + +1. ``$HOME/.reframe/settings.{py,json}`` +2. ``$RFM_INSTALL_PREFIX/settings.{py,json}`` +3. ``/etc/reframe.d/settings.{py,json}`` + +ReFrame accepts configuration files either in Python or JSON syntax. +If both are found in the same location, the Python file will be preferred. + +The ``RFM_INSTALL_PREFIX`` environment variable refers to the installation directory of ReFrame. +Users have no control over this variable. +It is always set by the framework upon startup. + +If no configuration file can be found in any of the predefined locations, ReFrame will fall back to a generic configuration that allows it to run on any system. +This configuration file is located in |reframe/core/settings.py|_. +Users may *not* modify this file. + +For a complete reference of the configuration, please refer to |reframe.settings(8)|_ man page. + +.. |reframe/core/settings.py| replace:: ``reframe/core/settings.py`` +.. _reframe/core/settings.py: https://github.com/eth-cscs/reframe/blob/master/reframe/core/settings.py +.. |reframe.settings(8)| replace:: ``reframe.settings(8)`` +.. _reframe.settings(8): config_reference.html + + +Reporting Bugs +-------------- + +For bugs, feature request, help, please open an issue on Github: + + +See Also +-------- + +See full documentation online: diff --git a/docs/manuals.rst b/docs/manuals.rst new file mode 100644 index 0000000000..bb8b2c6542 --- /dev/null +++ b/docs/manuals.rst @@ -0,0 +1,12 @@ +=============== +ReFrame Manuals +=============== + + +.. toctree:: + :maxdepth: 2 + + manpage + config_reference + regression_test_api + sanity_functions_reference diff --git a/docs/pipeline.rst b/docs/pipeline.rst index 38820dd219..f50060ee77 100644 --- a/docs/pipeline.rst +++ b/docs/pipeline.rst @@ -1,130 +1,131 @@ -============================ +========================== +How ReFrame Executes Tests +========================== + +A ReFrame test will be normally tried for different programming environments and different partitions within the same ReFrame run. +These are defined in the test's :func:`__init__` method, but it is not this original test object that is scheduled for execution. +The following figure explains in more detail the process: + +.. figure:: _static/img/reframe-test-cases.svg + :align: center + :alt: How ReFrame loads and schedules tests for execution. + + :sub:`How ReFrame loads and schedules tests for execution.` + +When ReFrame loads a test from the disk it unconditionally constructs it executing its :func:`__init__` method. +The practical implication of this is that your test will be instantiated even if it will not run on the current system. +After all the tests are loaded, they are filtered based on the current system and any other criteria (such as programming environment, test attributes etc.) specified by the user (see `Test Filtering `__ for more details). +After the tests are filtered, ReFrame creates the actual `test cases` to be run. A test case is essentially a tuple consisting of the test, the system partition and the programming environment to try. +The test that goes into a test case is essentially a `clone` of the original test that was instantiated upon loading. +This ensures that the test case's state is not shared and may not be reused in any case. +Finally, the generated test cases are passed to a `runner` that is responsible for scheduling them for execution based on the selected execution policy. + + The Regression Test Pipeline -============================ +---------------------------- -The backbone of the ReFrame regression framework is the regression test pipeline. -This is a set of well defined phases that each regression test goes through during its lifetime. -The figure below depicts this pipeline in detail. +Each ReFrame test case goes through a pipeline with clearly defined stages. +ReFrame tests can customize their operation as they execute by attaching hooks to the pipeline stages. +The following figure shows the different pipeline stages. .. figure:: _static/img/pipeline.svg :align: center :alt: The regression test pipeline + :sub:`The regression test pipeline.` - The regression test pipeline -A regression test starts its life after it has been instantiated by the framework. -This is where all the basic information of the test is set. -At this point, although it is initialized, the regression test is not yet *live*, meaning that it does not run yet. -The framework will then go over all the loaded and initialized checks (we will talk about the loading and selection phases later), it will pick the next partition of the current system and the next programming environment for testing and will try to run the test. -If the test supports the current system partition and the current programming environment, it will be run and it will go through all the following seven phases: +All tests will go through every stage one after the other. +However, some types of tests implement some stages as no-ops, whereas the sanity or performance check phases may be skipped on demand (see :option:`--skip-sanity-check` and :option:`--skip-performance-check` options). +In the following we describe in more detail what happens in every stage. -1. Setup -2. Compilation -3. Running -4. Sanity checking -5. Performance checking -6. Cleanup +--------------- +The Setup Phase +--------------- -A test may implement some of them as no-ops. As soon as the test is finished, its resources are cleaned up and the framework's environment is restored. -ReFrame will try to repeat the same procedure on the same regression test using the next programming environment and the next system partition until no further environments and partitions are left to be tested. -In the following we elaborate on each of the individual phases of the lifetime of a regression test. +During this phase the test will be set up for the currently selected system partition and programming environment. +The :attr:`current_partition` and :attr:`current_environ` test attributes will be set and the paths associated to this test case (stage, output and performance log directories) will be created. +A `job descriptor `__ will also be created for the test case containing information about the job to be submitted later in the pipeline. -0. The Initialization Phase ---------------------------- -This phase is not part of the regression test pipeline as shown above, but it is quite important, since during this phase the test is loaded into memory and initialized. -As we shall see in the `"Tutorial" `__ and in the `"Customizing Further A ReFrame Regression Test" `__ sections, this is the phase where the *specification* of a test is set. -At this point the current system is already known and the test may be set up accordingly. -If no further differentiation is needed depending on the system partition or the programming environment, the test could go through the whole pipeline performing all of its work without the need to override any of the other pipeline stages. -In fact, this is perhaps the most common case for most of the regression tests. +--------------- +The Build Phase +--------------- -1. The Setup Phase ------------------- +During this phase the source code associated with the test is compiled using the current programming environment. +If the test is `"run-only," `__ this phase is a no-op. -A regression test is instantiated once by the framework and it is then copied each time a new system partition or programming environment is tried. -This first phase of the regression pipeline serves the purpose of preparing the test to run on the specified partition and programming environment by performing a number of operations described below: +Before building the test, all the `resources `__ associated with it are copied to the test case's stage directory. +ReFrame then temporarily switches to that directory and builds the test. -Set up and load the test's environment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +------------- +The Run Phase +------------- -At this point the environment of the current partition, the current programming environment and any test's specific environment will be loaded. -For example, if the current partition requires ``slurm``, the current programming environment is ``PrgEnv-gnu`` and the test requires also ``cudatoolkit``, this phase will be equivalent to the following: +During this phase a job script associated with the test case will be created and it will be submitted for execution. +If the test is `"run-only," `__ its `resources `__ will be first copied to the test case's stage directory. +ReFrame will temporarily switch to that directory and spawn the test's job from there. +This phase is executed asynchronously (either a batch job is spawned or a local process is started) and it is up to the selected `execution policy <#execution-policies>`__ to block or not until the associated job finishes. -.. code:: bash - module load slurm - module unload PrgEnv-cray - module load PrgEnv-gnu - module load cudatoolkit +---------------- +The Sanity Phase +---------------- -Note that the framework automatically detects conflicting modules and unloads them first. -So the user need not to care about the existing environment at all. -She only needs to specify what is needed by her test. +During this phase, the sanity of the test's output is checked. +ReFrame makes no assumption as of what a successful test is; it does not even look into its exit code. +This is entirely up to the test to define. +ReFrame provides a flexible and expressive way for specifying complex patterns and operations to be performed on the test's output in order to determine the outcome of the test. -Setup the test's paths -^^^^^^^^^^^^^^^^^^^^^^ +--------------------- +The Performance Phase +--------------------- -Each regression test is associated with a stage directory and an output directory. -The stage directory will be the working directory of the test and all of its resources will be copied there before running. -The output directory is the directory where some important output files of the test will be kept. -By default these are the generated job script file, the standard output and standard error. -The user can also specify additional files to be kept in the test's specification. -At this phase, all these directories are created. +During this phase, the performance metrics reported by the test (if it is performance test) are collected, logged and compared to their reference values. +The mechanism for extracting performance metrics from the test's output is the same used by the sanity checking phase for extracting patterns from the test's output. -Prepare a job for the test -^^^^^^^^^^^^^^^^^^^^^^^^^^ +----------------- +The Cleanup Phase +----------------- -At this point a *job descriptor* will be created for the test. -A job descriptor in ReFrame is an abstraction of the job scheduler's functionality relevant to the regression framework. -It is responsible for submitting a job in a job queue and waiting for its completion. -ReFrame supports two job scheduler backends that can be combined with several different parallel program launchers. -For a complete list of the job scheduler/parallel launchers combinations, please refer to `"Partition Configuration" `__. +During this final stage of the pipeline, the test's resources are cleaned up. +More specifically, if the test has finished successfully, all interesting test files (build/job scripts, build/job script output and any user-specified files) are copied to ReFrame's output directory and the stage directory of the test is deleted. -2. The Compilation Phase ------------------------- -At this phase the source code associated with test is compiled with the current programming environment. -Before compiling, all the resources of the test are copied to its stage directory and the compilation is performed from that directory. +Execution Policies +------------------ -3. The Run Phase ----------------- +All regression tests in ReFrame will execute the pipeline stages described above. +However, how exactly this pipeline will be executed is responsibility of the test execution policy. +There are two execution policies in ReFrame: the serial and the asynchronous one. -This phase comprises two subphases: +In the serial execution policy, a new test gets into the pipeline after the previous one has exited. +As the figure below shows, this can lead to long idling times in the run phase, since the execution blocks until the associated test job finishes. -* **Job launch**: At this subphase a job script file for the regression test is generated and submitted to the job scheduler queue. - If the job scheduler for the current partition is the **local** one, a simple wrapper shell script will be generated and will be launched as a local OS process. -* **Job wait**: At this subphase the job (or local process) launched in the previous subphase is waited for. - This phase is pretty basic: it just checks that the launched job (or local process) has finished. - No check is made of whether the job or process has finished successfully or not. - This is the responsibility of the next pipeline stage. -ReFrame currently supports two execution policies: +.. figure:: _static/img/serial-exec-policy.svg + :align: center + :alt: The serial execution policy. -* **serial**: In the serial execution policy, these two subphases are performed back-to-back and the framework blocks until the current regression test finishes. -* **asynchronous**: In the asynchronous execution policy, as soon as the job associated to the current test is launched, ReFrame continues its execution by executing and launching the subsequent test cases. + :sub:`The serial execution policy.` -4. The Sanity Checking Phase ----------------------------- -At this phase it is determined whether the check has finished successfully or not. -Although this decision is test-specific, ReFrame provides a very flexible and expressive way for specifying complex patterns and operations to be performed on the test's output in order to determine the outcome of the test. +In the asynchronous execution policy, multiple tests can be simultaneously on-the-fly. +When a test enters the run phase, ReFrame does not block, but continues by picking the next test case to run. +This continues until no more test cases are left for execution or until a maximum concurrency limit is reached. +At the end, ReFrame enters a busy-wait loop monitoring the spawned test cases. +As soon as test case finishes, it resumes its pipeline and runs it to completion. +The following figure shows how the asynchronous execution policy works. -5. The Performance Checking Phase ---------------------------------- -At this phase the performance of the regression test is checked. -ReFrame uses the same mechanism for analyzing the output of the test as with sanity checking. -The only difference is that the user can now specify reference values per system or system partition, as well as acceptable performance thresholds +.. figure:: _static/img/async-exec-policy.svg + :align: center + :alt: The asynchronous execution policy. -6. The Cleanup Phase --------------------- + :sub:`The asynchronous execution policy.` -This is the final stage of the regression test pipeline and it is responsible for cleaning up the resources of the test. -Three steps are performed in this phase: -1. The interesting files of the test (job script, standard output and standard error and any additional files specified by the user) are copied to its output directory for later inspection and bookkeeping, -2. the stage directory is removed and -3. the test's environment is revoked. +ReFrame tries to keep concurrency high by maintaining as many test cases as possible simultaneously active. +When the `concurrency limit `__ is reached, ReFrame will first try to free up execution slots by checking if any of the spawned jobs have finished, and it will fill that slots first before throttling execution. -At this point the ReFrame's environment is clean and in its original state and the framework may continue by running more test cases. +ReFrame uses polling to check the status of the spawned jobs, but it does so in a dynamic way, in order to ensure both responsiveness and avoid overloading the system job scheduler with excessive polling. diff --git a/docs/reference.rst b/docs/regression_test_api.rst similarity index 61% rename from docs/reference.rst rename to docs/regression_test_api.rst index 43b43ea583..989f1eb408 100644 --- a/docs/reference.rst +++ b/docs/regression_test_api.rst @@ -1,29 +1,45 @@ -=============== -Reference Guide -=============== +======================== +ReFrame Programming APIs +======================== This page provides a reference guide of the ReFrame API for writing regression tests covering all the relevant details. -Internal data structures and APIs are covered only to the extent that might be helpful to the final user of the framework. +Internal data structures and APIs are covered only to the extent that this might be helpful to the final user of the framework. -Regression test classes and related utilities ---------------------------------------------- - -.. automodule:: reframe.core.decorators - :members: - :show-inheritance: +Regression Test Base Classes +---------------------------- .. automodule:: reframe.core.pipeline :members: :show-inheritance: +Regression Test Class Decorators +-------------------------------- + +.. autodecorator:: reframe.core.decorators.parameterized_test(*inst) + +.. autodecorator:: reframe.core.decorators.required_version(*versions) + +.. autodecorator:: reframe.core.decorators.simple_test + + +Pipeline Hooks +-------------- + +.. autodecorator:: reframe.core.decorators.run_after(stage) + +.. autodecorator:: reframe.core.decorators.run_before(stage) + +.. autodecorator:: reframe.core.decorators.require_deps + + Environments and Systems ------------------------ .. automodule:: reframe.core.environments - :members: + :members: Environment, ProgEnvironment, _EnvironmentSnapshot, snapshot :show-inheritance: .. automodule:: reframe.core.systems @@ -31,10 +47,10 @@ Environments and Systems :show-inheritance: -Job schedulers and parallel launchers +Job Schedulers and Parallel Launchers ------------------------------------- -.. autoclass:: reframe.core.schedulers.Job +.. automodule:: reframe.core.schedulers :members: :show-inheritance: @@ -43,27 +59,38 @@ Job schedulers and parallel launchers :show-inheritance: -.. automodule:: reframe.core.backends - :members: - :show-inheritance: +.. autofunction:: reframe.core.backends.getlauncher(name) + + Retrieve the :class:`reframe.core.launchers.JobLauncher` concrete + implementation for a parallel launcher backend. + + :arg name: The registered name of the launcher backend. -Runtime services +.. autofunction:: reframe.core.backends.getscheduler(name) + + Retrieve the :class:`reframe.core.schedulers.JobScheduler` concrete + implementation for a scheduler backend. + + :arg name: The registered name of the scheduler backend. + +Runtime Services ---------------- .. automodule:: reframe.core.runtime :members: - :exclude-members: temp_runtime, switch_runtime :show-inheritance: -Modules System API ------------------- +Modules Systems +--------------- .. autoclass:: reframe.core.modules.ModulesSystem + :members: + :show-inheritance: -Build systems +Build Systems ------------- .. versionadded:: 2.14 @@ -92,16 +119,11 @@ It is up to the concrete build system implementations on how to use or not these :show-inheritance: -Container platforms +Container Platforms ------------------- .. versionadded:: 2.20 -ReFrame can run a regression test inside a container. -To achieve that you have to set the :attr:`reframe.core.pipeline.RegressionTest.container_platform` attribute and then set up the container platform (e.g., image to load, commands to execute). -The :class:`reframe.core.ContainerPlatform` abstract base class define the basic interface and a minimal set of attributes that all concrete container platforms must implement. -Concrete container platforms may also define additional fields that are specific to them. - .. automodule:: reframe.core.containers :members: :exclude-members: ContainerPlatformField diff --git a/docs/requirements.txt b/docs/requirements.txt index a9bb99f4c2..4b29a76882 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -18,7 +18,7 @@ requests>=2.18.4 setuptools>=31.0.1 six>=1.11.0 snowballstemmer>=1.2.1 -Sphinx>=1.7.5 +Sphinx>=3.0 sphinx-autobuild>=0.7.1 sphinx-bootstrap-theme>=0.5.3 sphinx-fakeinv>=1.0.0 diff --git a/docs/running.rst b/docs/running.rst deleted file mode 100644 index b266667c26..0000000000 --- a/docs/running.rst +++ /dev/null @@ -1,1122 +0,0 @@ -=============== -Running ReFrame -=============== - -Before getting into any details, the simplest way to invoke ReFrame is the following: - -.. code-block:: bash - - ./bin/reframe -c /path/to/checks -R -r - -This will search recursively for test files in ``/path/to/checks`` and will start running them on the current system. - -ReFrame's front-end goes through three phases: - -1. Load tests -2. Filter tests -3. Act on tests - -In the following, we will elaborate on these phases and the key command-line options controlling them. -A detailed listing of all the command-line options grouped by phase is given by ``./bin/reframe -h``. - -Supported Actions ------------------ - -Even though an action is the last phase that the front-end goes through, we are listing it first since an action is always required. -Currently there are only two available actions: - -1. Listing of the selected checks -2. Execution of the selected checks - -------------------------------- -Listing of the regression tests -------------------------------- - -To retrieve a listing of the selected checks, you must specify the ``-l`` or ``--list`` options. -This will provide a list with a brief description for each test containing only its name and the path to the file where it is defined. -An example listing of checks is the following that lists all the tests found under the ``tutorial/`` folder: - -.. code-block:: bash - - ./bin/reframe -c tutorial -l - -The output looks like: - -.. code-block:: none - - Command line: ./bin/reframe -c tutorial/ -l - Reframe version: 2.15-dev1 - Launched by user: USER - Launched on host: daint103 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial/' - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Logging dir : /path/to/reframe/logs - List of matched checks - ====================== - * Example5Test (found in /path/to/reframe/tutorial/example5.py) - * Example1Test (found in /path/to/reframe/tutorial/example1.py) - * Example4Test (found in /path/to/reframe/tutorial/example4.py) - * SerialTest (found in /path/to/reframe/tutorial/example8.py) - * OpenMPTest (found in /path/to/reframe/tutorial/example8.py) - * MPITest (found in /path/to/reframe/tutorial/example8.py) - * OpenACCTest (found in /path/to/reframe/tutorial/example8.py) - * CudaTest (found in /path/to/reframe/tutorial/example8.py) - * Example3Test (found in /path/to/reframe/tutorial/example3.py) - * Example7Test (found in /path/to/reframe/tutorial/example7.py) - * Example6Test (found in /path/to/reframe/tutorial/example6.py) - * Example2aTest (found in /path/to/reframe/tutorial/example2.py) - * Example2bTest (found in /path/to/reframe/tutorial/example2.py) - Found 13 check(s). - -You may also retrieve a listing with detailed information about the each check using the option ``-L`` or ``--list-detailed``. -The following example lists detailed information about the tutorial check: - -.. code-block:: none - - Command line: ./bin/reframe -c tutorial/ -L - Reframe version: 2.18-dev2 - Launched by user: USER - Launched on host: daint103 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial/' - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Logging dir : /path/to/reframe/logs - List of matched checks - ====================== - * Example5Test (found in /path/to/reframe/tutorial/example5.py) - - description: Matrix-vector multiplication example with CUDA - - systems: daint:gpu - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-pgi - - modules: cudatoolkit - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example1Test (found in /path/to/reframe/tutorial/example1.py) - - description: Simple matrix-vector multiplication example - - systems: * - - environments: * - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example4Test (found in /path/to/reframe/tutorial/example4.py) - - description: Matrix-vector multiplication example with OpenACC - - systems: daint:gpu - - environments: PrgEnv-cray, PrgEnv-pgi - - modules: craype-accel-nvidia60 - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * SerialTest (found in /path/to/reframe/tutorial/example8.py) - - description: Serial matrix-vector multiplication - - systems: * - - environments: * - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * OpenMPTest (found in /path/to/reframe/tutorial/example8.py) - - description: OpenMP matrix-vector multiplication - - systems: * - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-intel, PrgEnv-pgi - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * MPITest (found in /path/to/reframe/tutorial/example8.py) - - description: MPI matrix-vector multiplication - - systems: daint:gpu, daint:mc - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-intel, PrgEnv-pgi - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * OpenACCTest (found in /path/to/reframe/tutorial/example8.py) - - description: OpenACC matrix-vector multiplication - - systems: daint:gpu - - environments: PrgEnv-cray, PrgEnv-pgi - - modules: craype-accel-nvidia60 - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * CudaTest (found in /path/to/reframe/tutorial/example8.py) - - description: CUDA matrix-vector multiplication - - systems: daint:gpu - - environments: PrgEnv-gnu, PrgEnv-cray, PrgEnv-pgi - - modules: cudatoolkit - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example3Test (found in /path/to/reframe/tutorial/example3.py) - - description: Matrix-vector multiplication example with MPI - - systems: daint:gpu, daint:mc - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-intel, PrgEnv-pgi - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example7Test (found in /path/to/reframe/tutorial/example7.py) - - description: Matrix-vector multiplication (CUDA performance test) - - systems: daint:gpu - - environments: PrgEnv-gnu, PrgEnv-cray, PrgEnv-pgi - - modules: cudatoolkit - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example6Test (found in /path/to/reframe/tutorial/example6.py) - - description: Matrix-vector multiplication with L2 norm check - - systems: * - - environments: * - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example2aTest (found in /path/to/reframe/tutorial/example2.py) - - description: Matrix-vector multiplication example with OpenMP - - systems: * - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-intel, PrgEnv-pgi - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - * Example2bTest (found in /path/to/reframe/tutorial/example2.py) - - description: Matrix-vector multiplication example with OpenMP - - systems: * - - environments: PrgEnv-cray, PrgEnv-gnu, PrgEnv-intel, PrgEnv-pgi - - modules: - - task allocation: standard - - tags: tutorial - - maintainers: you-can-type-your-email-here - Found 13 check(s). - - -The detailed listing shows the description of the test, its supported systems and programming environments (``*`` stands for any system or programming environment), the environment modules that it loads, its tags and its maintainers. - -.. warning:: - The list of modules showed in the detailed listing may not correspond to actual modules loaded by test, if the test customizes its behavior during the setup stage. - - -.. note:: - .. versionadded:: 2.15 - - Support for detailed listings. - Standard listing using the ``-l`` option is now shorter. - -.. note:: - .. versionchanged:: 2.15 - - Test listing lists only tests supported by the current system. - Previous versions were showing all the tests found. - - ---------------------------------- -Execution of the regression tests ---------------------------------- - -To run the regression tests you should specify the *run* action though the ``-r`` or ``--run`` options. - -.. note:: The listing action takes precedence over the execution, meaning that if you specify both ``-l -r``, only the listing action will be performed. - - -.. code-block:: bash - - ./reframe.py -C tutorial/config/settings.py -c tutorial/example1.py -r - -The output of the regression run looks like the following: - -.. code-block:: none - - Command line: ./reframe.py -C tutorial/config/settings.py -c tutorial/example1.py -r - Reframe version: 3.0-dev3 (rev: 0c62d00c) - Launched by user: USER - Launched on host: daint105 - Reframe paths - ============= - Check prefix : - Check search path : '/path/to/reframe/tutorial/example1.py' - Current working dir : /path/to/reframe - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Perf. logging prefix : /path/to/reframe/perflogs - [==========] Running 1 check(s) - [==========] Started on Wed Mar 25 10:13:13 2020 - - [----------] started processing Example1Test (Simple matrix-vector multiplication example) - [ RUN ] Example1Test on daint:login using PrgEnv-cray - [ RUN ] Example1Test on daint:login using PrgEnv-gnu - [ RUN ] Example1Test on daint:login using PrgEnv-intel - [ RUN ] Example1Test on daint:login using PrgEnv-pgi - [ RUN ] Example1Test on daint:gpu using PrgEnv-cray - [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu - [ RUN ] Example1Test on daint:gpu using PrgEnv-intel - [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi - [ RUN ] Example1Test on daint:mc using PrgEnv-cray - [ RUN ] Example1Test on daint:mc using PrgEnv-gnu - [ RUN ] Example1Test on daint:mc using PrgEnv-intel - [ RUN ] Example1Test on daint:mc using PrgEnv-pgi - [----------] finished processing Example1Test (Simple matrix-vector multiplication example) - - [----------] waiting for spawned checks to finish - [ OK ] ( 1/12) Example1Test on daint:login using PrgEnv-intel - [ OK ] ( 2/12) Example1Test on daint:login using PrgEnv-cray - [ OK ] ( 3/12) Example1Test on daint:login using PrgEnv-gnu - [ OK ] ( 4/12) Example1Test on daint:login using PrgEnv-pgi - [ OK ] ( 5/12) Example1Test on daint:mc using PrgEnv-gnu - [ OK ] ( 6/12) Example1Test on daint:mc using PrgEnv-pgi - [ OK ] ( 7/12) Example1Test on daint:mc using PrgEnv-cray - [ OK ] ( 8/12) Example1Test on daint:mc using PrgEnv-intel - [ OK ] ( 9/12) Example1Test on daint:gpu using PrgEnv-intel - [ OK ] (10/12) Example1Test on daint:gpu using PrgEnv-cray - [ OK ] (11/12) Example1Test on daint:gpu using PrgEnv-gnu - [ OK ] (12/12) Example1Test on daint:gpu using PrgEnv-pgi - [----------] all spawned checks have finished - - [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Wed Mar 25 10:21:08 2020 - - -Discovery of Regression Tests ------------------------------ - -When ReFrame is invoked, it tries to locate regression tests in a predefined path. -By default, this path is the ``/checks``. -You can also retrieve this path as follows: - -.. code-block:: bash - - ./bin/reframe -l | grep 'Check search path' - -If the path line is prefixed with ``(R)``, every directory in that path will be searched recursively for regression tests. - -As described extensively in the `"ReFrame Tutorial" `__, regression tests in ReFrame are essentially Python source files that provide a special function, which returns the actual regression test instances. -A single source file may also provide multiple regression tests. -ReFrame loads the python source files and tries to call this special function; -if this function cannot be found, the source file will be ignored. -At the end of this phase, the front-end will have instantiated all the tests found in the path. - -You can override the default search path for tests by specifying the ``-c`` or ``--checkpath`` options. -We have already done that already when listing all the tutorial tests: - -.. code-block:: bash - - ./bin/reframe -c tutorial/ -l - -ReFrame the does not search recursively into directories specified with the ``-c`` option, unless you explicitly specify the ``-R`` or ``--recurse`` options. - -The ``-c`` option completely overrides the default path. -Currently, there is no option to prepend or append to the default regression path. -However, you can build your own check path by specifying multiple times the ``-c`` option. -The ``-c`` option accepts also regular files. -This is very useful when you are implementing new regression tests, since it allows you to run only your test: - -.. code-block:: bash - - ./bin/reframe -c /path/to/my/new/test.py -r - -.. important:: - The names of the loaded tests must be unique. - Trying to load two or more tests with the same name will produce an error. - You may ignore the error by using the ``--ignore-check-conflicts`` option. - In this case, any conflicting test will not be loaded and a warning will be issued. - - .. versionadded:: 2.12 - - - -Filtering of Regression Tests ------------------------------ - -At this phase you can select which regression tests should be run or listed. -There are several ways to select regression tests, which we describe in more detail here: - -------------------------- -Selecting tests by system -------------------------- - -.. versionadded:: 2.15 - - -By default, ReFrame always selects the tests that are supported by the current system. -If you want to list the tests supported by a different system, you may achieve that by passing the ``--system`` option: - -.. code-block:: bash - - ./bin/reframe --system=kesch -l - - -This example lists all the tests that are supported by the system named ``kesch``. -It is also possible to list only the tests that are supported by a specific system partition. -The following example will list only the tests suported by the ``login`` partition of the ``kesch`` system: - -.. code-block:: bash - - ./bin/reframe --system=kesch:login -l - - -Finally, in order to list all the tests found regardless of their supported systems, you should pass the ``--skip-system-check`` option: - -.. code-block:: bash - - ./bin/reframe --skip-system-check -l - - ------------------------------------------- -Selecting tests by programming environment ------------------------------------------- - -To select tests by the programming environment, use the ``-p`` or ``--prgenv`` options: - -.. code-block:: bash - - ./bin/reframe -p PrgEnv-gnu -l - -This will select all the checks that support the ``PrgEnv-gnu`` environment. - -You can also specify multiple times the ``-p`` option, in which case a test will be selected if it support all the programming environments specified in the command line. -For example the following will select all the checks that can run with both ``PrgEnv-cray`` and ``PrgEnv-gnu`` on the current system: - -.. code-block:: bash - - ./bin/reframe -p PrgEnv-gnu -p PrgEnv-cray -l - -If you are going to run a set of tests selected by programming environment, they will run only for the selected programming environment(s). - -The ``-p`` option accepts also the Python `regular expression syntax `__. -In fact, the argument to ``-p`` option is treated as a regular expression always. This means that the ``-p PrgEnv-gnu`` will match also tests that support a ``PrgEnv-gnuXX`` environment. -If you would like to stricly select tests that support ``PrgEnv-gnu`` only and not ``PrgEnv-gnuXX``, you should write ``-p PrgEnv-gnu$``. -As described above multiple ``-p`` options are AND-ed. -Combining that with regular expressions can be quite powerful. -For example, the following will select all tests that support programming environment ``foo`` and either ``PrgEnv-gnu`` or ``PrgEnv-pgi``: - -.. code-block:: bash - - ./bin/reframe -p foo -p 'PrgEnv-(gnu|pgi)' -l - - -.. note:: - .. versionadded:: 2.17 - - The ``-p`` option recognizes regular expressions as arguments. - - ------------------------ -Selecting tests by tags ------------------------ - -As we have seen in the `"ReFrame tutorial" `__, every regression test may be associated with a set of tags. -Using the ``-t`` or ``--tag`` option you can select the regression tests associated with a specific tag. -For example the following will list all the tests that have a ``maintenance`` tag and can run on the current system: - -.. code-block:: bash - - ./bin/reframe -t maintenance -l - -Similarly to the ``-p`` option, you can chain multiple ``-t`` options together, in which case a regression test will be selected if it is associated with all the tags specified in the command line. -The list of tags associated with a check can be viewed in the listing output when specifying the ``-l`` option. - -.. note:: - .. versionadded:: 2.17 - - The ``-t`` option recognizes regular expressions as arguments. - ------------------------ -Selecting tests by name ------------------------ - -It is possible to select or exclude tests by name through the ``--name`` or ``-n`` and ``--exclude`` or ``-x`` options. -For example, you can select only the ``Example7Test`` from the tutorial as follows: - -.. code-block:: bash - - ./bin/reframe -c tutorial/ -n Example7Test -l - -.. code-block:: none - - Command line: ./bin/reframe -c tutorial/ -n Example7Test -l - Reframe version: 2.15-dev1 - Launched by user: USER - Launched on host: daint103 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial' - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Logging dir : /path/to/reframe/logs - List of matched checks - ====================== - * Example7Test (found in /path/to/reframe/tutorial/example7.py) - Found 1 check(s). - - -Similarly, you can exclude this test by passing the ``-x Example7Test`` option: - -.. code-block:: none - - Command line: ./bin/reframe -c tutorial -x Example7Test -l - Reframe version: 2.15-dev1 - Launched by user: USER - Launched on host: daint103 - Reframe paths - ============= - Check prefix : - Check search path : 'tutorial' - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Logging dir : /path/to/reframe/logs - List of matched checks - ====================== - * Example5Test (found in /path/to/reframe/tutorial/example5.py) - * Example1Test (found in /path/to/reframe/tutorial/example1.py) - * Example4Test (found in /path/to/reframe/tutorial/example4.py) - * SerialTest (found in /path/to/reframe/tutorial/example8.py) - * OpenMPTest (found in /path/to/reframe/tutorial/example8.py) - * MPITest (found in /path/to/reframe/tutorial/example8.py) - * OpenACCTest (found in /path/to/reframe/tutorial/example8.py) - * CudaTest (found in /path/to/reframe/tutorial/example8.py) - * Example3Test (found in /path/to/reframe/tutorial/example3.py) - * Example6Test (found in /path/to/reframe/tutorial/example6.py) - * Example2aTest (found in /path/to/reframe/tutorial/example2.py) - * Example2bTest (found in /path/to/reframe/tutorial/example2.py) - Found 12 check(s). - - -Both ``-n`` and ``-x`` options can be chained, in which case either the tests that have any of the specified names are selected or excluded from running. -They may also accept regular expressions as arguments. - -.. note:: - .. versionadded:: 2.17 - - The ``-n`` and ``-x`` options recognize regular expressions as arguments. - Chaining these options, e.g., ``-n A -n B``, is equivalent to a regular expression that applies OR to the individual arguments, i.e., equivalent to ``-n 'A|B'``. - - - - -Controlling the Execution of Regression Tests ---------------------------------------------- - -There are several options for controlling the execution of regression tests. -Keep in mind that these options will affect all the tests that will run with the current invocation. -They are summarized below: - -* ``-A ACCOUNT``, ``--account ACCOUNT``: Submit regression test jobs using ``ACCOUNT``. -* ``-P PART``, ``--partition PART``: Submit regression test jobs in the *scheduler partition* ``PART``. -* ``--reservation RES``: Submit regression test jobs in reservation ``RES``. -* ``--nodelist NODELIST``: Run regression test jobs on the nodes specified in ``NODELIST``. -* ``--exclude-nodes NODELIST``: Do not run the regression test jobs on any of the nodes specified in ``NODELIST``. -* ``--job-option OPT``: Pass option ``OPT`` directly to the back-end job scheduler. This option *must* be used with care, since you may break the submission mechanism. - All of the above job submission related options could be expressed with this option. - For example, the ``-n NODELIST`` is equivalent to ``--job-option='--nodelist=NODELIST'`` for a Slurm job scheduler. - If you pass an option that is already defined by the framework, the framework will *not* explicitly override it; this is up to scheduler. - All extra options defined from the command line are appended to the automatically generated options in the generated batch script file. - So if you redefine one of them, e.g., ``--output`` for the Slurm scheduler, it is up the job scheduler on how to interpret multiple definitions of the same options. - In this example, Slurm's policy is that later definitions of options override previous ones. - So, in this case, way you would override the standard output for all the submitted jobs! - -* ``--flex-alloc-tasks {all|idle|NUM}``: (Deprecated) Please use ``--flex-alloc-nodes`` instead. -* ``--flex-alloc-nodes {all|idle|NUM}``: Automatically determine the number of nodes allocated for each test. -* ``--force-local``: Force the local execution of the selected tests. - No jobs will be submitted. -* ``--skip-sanity-check``: Skip sanity checking phase. -* ``--skip-performance-check``: Skip performance verification phase. -* ``--strict``: Force strict performance checking. Some tests may set their :attr:`strict_check ` attribute to :class:`False` (see `"Reference Guide" `__) in order to just let their performance recorded but not yield an error. - This option overrides this behavior and forces all tests to be strict. -* ``--skip-system-check``: Skips the system check and run the selected tests even if they do not support the current system. - This option is sometimes useful when you need to quickly verify if a regression test supports a new system. -* ``--skip-prgenv-check``: Skips programming environment check and run the selected tests for even if they do not support a programming environment. - This option is useful when you need to quickly verify if a regression check supports another programming environment. - For example, if you know that a tests supports only ``PrgEnv-cray`` and you need to check if it also works with ``PrgEnv-gnu``, you can test is as follows: - - .. code-block:: bash - - ./bin/reframe -c /path/to/my/check.py -p PrgEnv-gnu --skip-prgenv-check -r - -* ``--max-retries NUM``: Specify the maximum number of times a failed regression test may be retried (default: 0). - - - -Generating a Performance Report -------------------------------- - -If you are running performance tests, you may instruct ReFrame to produce a performance report at the end using the `--performance-report` command-line options. -The performance report is printed after the output of the regression tests and has the following format: - -.. code-block:: none - - PERFORMANCE REPORT - ------------------------------------------------------------------------------ - Check1 - - system:partition - - PrgEnv1 - * num_tasks: - * perf_variable1: - * perf_variable2: - * ... - - PrgEnv2 - * num_tasks: - * perf_variable1: - * perf_variable2: - * ... - ------------------------------------------------------------------------------ - Check2 - - system:partition - - PrgEnv1 - * num_tasks: - * perf_variable1: - * perf_variable2: - * ... - - PrgEnv2 - * num_tasks: - * perf_variable1: - * perf_variable2: - * ... - ------------------------------------------------------------------------------ - -The number of tasks and the achieved performance values are listed by system partition and programming environment for each performance test that has run. -Performance variables are the variables collected through the :attr:`reframe.core.pipeline.RegressionTest.perf_patterns` attribute. - -The following command will run the CUDA matrix-vector multiplication example from the `tutorial `__ and will produce a performance report: - -.. code-block:: bash - - ./bin/reframe -C tutorial/config/settings.py -c tutorial/example7.py -r --performance-report - -.. code-block:: none - - Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/example7.py -r --performance-report - Reframe version: 3.0-dev3 (rev: 0c62d00c) - Launched by user: USER - Launched on host: daint105 - Reframe paths - ============= - Check prefix : - Check search path : '/path/to/reframe/tutorial/example7.py' - Current working dir : /path/to/reframe - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Perf. logging prefix : /path/to/reframe/perflogs - [==========] Running 1 check(s) - [==========] Started on Wed Mar 25 10:40:25 2020 - - [----------] started processing Example7Test (Matrix-vector multiplication (CUDA performance test)) - [ RUN ] Example7Test on daint:gpu using PrgEnv-cray - [ RUN ] Example7Test on daint:gpu using PrgEnv-gnu - [ RUN ] Example7Test on daint:gpu using PrgEnv-pgi - [----------] finished processing Example7Test (Matrix-vector multiplication (CUDA performance test)) - - [----------] waiting for spawned checks to finish - [ OK ] (1/3) Example7Test on daint:gpu using PrgEnv-gnu - [ OK ] (2/3) Example7Test on daint:gpu using PrgEnv-cray - [ OK ] (3/3) Example7Test on daint:gpu using PrgEnv-pgi - [----------] all spawned checks have finished - - [ PASSED ] Ran 3 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Wed Mar 25 10:43:53 2020 - ============================================================================== - PERFORMANCE REPORT - ------------------------------------------------------------------------------ - Example7Test - - daint:gpu - - PrgEnv-cray - * num_tasks: 1 - * perf: 50.184982 Gflop/s - - PrgEnv-gnu - * num_tasks: 1 - * perf: 50.205857 Gflop/s - - PrgEnv-pgi - * num_tasks: 1 - * perf: 49.691865 Gflop/s - ------------------------------------------------------------------------------ - - -For completeness, we show here the corresponding section from the ``Example7Test``, so that the connection between the test's code and the output becomes clear: - -.. literalinclude:: ../tutorial/example7.py - :lines: 19-27 - :dedent: 8 - - -If you are writing a benchmark, it is often the case that you will run it in an unknown system, where you don't have any reference value. -Normally, if ReFrame cannot find a reference for the system it is running on, it will complain and mark the test as a failure. -However, you may right your test in such a way, that it allows it to run successfully on any new system. -To achieve this, simply insert a "catch-all" ``*`` entry in the :attr:`reframe.core.pipeline.RegressionTest.reference` attribute: - - -.. code-block:: python - - self.reference = { - '*': { - 'perf_var1': (0, None, None, 'units'), - 'perf_var2': (0, None, None, 'units') - ... - } - } - -The performance test will always pass on new systems and you may use the ``--performance-report`` option for getting the actual performance values. - - -.. note:: - - The performance report should not be confused with `performance logging <#performance-logging>`__. - It is simply a way of quickly visualizing the performance results and is useful for interactive testing. - Performance logging, if configured, occurs independently of the performance report and is meant for keeping performance data over time. - Its formatting facilitates parsing and it should be used for later analysis of the performance data obtained. - - -Configuring ReFrame Directories -------------------------------- - -ReFrame uses two basic directories during the execution of tests: - -1. The stage directory - - * Each regression test is executed in a "sandbox"; - all of its resources (source files, input data etc.) are copied over to a stage directory (if the directory preexists, it will be wiped out) and executed from there. - This will also be the working directory for the test. - -2. The output directory - - * After a regression test finishes some important files will be copied from the stage directory to the output directory (if the directory preexists, it will be wiped out). - By default these are the standard output, standard error and the generated job script file. - A regression test may also specify to keep additional files. - -By default, these directories are placed under a common prefix, which defaults to ``.``. -The rest of the directories are organized as follows: - -* Stage directory: ``${prefix}/stage/`` -* Output directory: ``${prefix}/output/`` - -You can optionally append a timestamp directory component to the above paths (except the logs directory), by using the ``--timestamp`` option. -This options takes an optional argument to specify the timestamp format. -The default `time format `__ is ``%FT%T``, which results into timestamps of the form ``2017-10-24T21:10:29``. - -You can override either the default global prefix or any of the default individual directories using the corresponding options. - -* ``--prefix DIR``: set prefix to ``DIR``. -* ``--output DIR``: set output directory to ``DIR``. -* ``--stage DIR``: set stage directory to ``DIR``. - -The stage and output directories are created only when you run a regression test. -However you can view the directories that will be created even when you do a listing of the available checks with the ``-l`` option. -This is useful if you want to check the directories that ReFrame will create. - -.. code-block:: bash - - ./bin/reframe -C tutorial/config/settings.py --prefix /foo -l - -.. code-block:: none - - Command line: ./bin/reframe -C tutorial/config/settings.py --prefix /foo -l - Reframe version: 2.13-dev0 - Launched by user: USER - Launched on host: daint103 - Reframe paths - ============= - Check prefix : /path/to/reframe - (R) Check search path : 'checks/' - Stage dir prefix : /foo/stage/ - Output dir prefix : /foo/output/ - Perf. logging prefix : /Users/karakasv/Repositories/reframe/logs - List of matched checks - ====================== - Found 0 check(s). - - -You can also define different default directories per system by specifying them in the `site configuration `__ settings file. -The command line options, though, take always precedence over any default directory. - - -Logging -------- - -From version 2.4 onward, ReFrame not only supports logging of its actions, but its normal output is also fully configurable. -All output of ReFrame is handled by logging handlers, which can send information to multiple channels at once filtering it independently based on severity levels. -By default, using its generic configuration, ReFrame sends informational data to its standard output and debug data to ``reframe.log``. -The debug information in ``reframe.log`` looks like the following: - -.. code-block:: none - - [2020-03-25T10:42:57] info: reframe: [ OK ] (1/3) Example7Test on daint:gpu using PrgEnv-gnu - [2020-03-25T10:42:57] debug: Example7Test: entering stage: cleanup - [2020-03-25T10:42:57] debug: Example7Test: copying interesting files to output directory - [2020-03-25T10:42:57] debug: Example7Test: removing stage directory - [2020-03-25T10:42:57] debug: reframe: polling rate (real): 0.614 polls/sec - [2020-03-25T10:42:57] debug: reframe: polling rate (desired): 0.200 - [2020-03-25T10:42:57] debug: reframe: sleeping: 10.000s - [2020-03-25T10:43:07] debug: reframe: running tasks: 2 - [2020-03-25T10:43:07] debug: reframe: updating counts for running test cases - [2020-03-25T10:43:07] debug: reframe: polling 2 task(s) - [2020-03-25T10:43:07] debug: Example7Test: entering stage: run - [2020-03-25T10:43:07] debug: Example7Test: executing OS command: sacct -S 2020-03-25 -P -j 21424988 -o jobid,state,ex - itcode,nodelist - [2020-03-25T10:43:07] debug: Example7Test: entering stage: run - [2020-03-25T10:43:07] debug: Example7Test: executing OS command: sacct -S 2020-03-25 -P -j 21424991 -o jobid,state,ex - itcode,nodelist - [2020-03-25T10:43:07] debug: reframe: finalizing tasks: 0 - [2020-03-25T10:43:07] debug: reframe: polling rate (real): 0.586 polls/sec - [2020-03-25T10:43:07] debug: reframe: polling rate (desired): 0.200 - [2020-03-25T10:43:07] debug: reframe: sleeping: 10.000s - [2020-03-25T10:43:17] debug: reframe: running tasks: 2 - [2020-03-25T10:43:17] debug: reframe: updating counts for running test cases - [2020-03-25T10:43:17] debug: reframe: polling 2 task(s) - [2020-03-25T10:43:17] debug: Example7Test: entering stage: run - [2020-03-25T10:43:17] debug: Example7Test: executing OS command: sacct -S 2020-03-25 -P -j 21424988 -o jobid,state,ex - itcode,nodelist - [2020-03-25T10:43:17] debug: Example7Test: entering stage: wait - [2020-03-25T10:43:17] debug: Example7Test: spawned job finished - [2020-03-25T10:43:17] debug: reframe: removing task from running list: Example7Test on daint:gpu using PrgEnv-cray - [2020-03-25T10:43:17] debug: reframe: finalizing tasks: 1 - [2020-03-25T10:43:17] debug: reframe: finalizing task: Example7Test on daint:gpu using PrgEnv-cray - [2020-03-25T10:43:17] debug: Example7Test: entering stage: sanity - [2020-03-25T10:43:17] debug: Example7Test: entering stage: performance - [2020-03-25T10:43:17] info: reframe: [ OK ] (2/3) Example7Test on daint:gpu using PrgEnv-cray - - -Each line starts with a timestamp, the level of the message (``info``, ``debug`` etc.), the context in which the framework is currently executing (either ``reframe`` or the name of the current test and, finally, the actual message. - -Every time ReFrame is run, the ``reframe.log`` files will be rewritten. -However, you can ask ReFrame to copy it to the output directory before exiting by passing it the ``--save-log-files`` option. - -Users have full control over what is being logged, where it is send and how log records are formatted. -The `configuration guide `__ gives an overview of the logging configuration, whereas a complete description of the available options can be found in the `configuration reference `__. - - -------------------- -Performance Logging -------------------- - -Performance testing is an essential part of ReFrame and performance data from tests is handled specially. -ReFrame allows you to send performance data to multiple channels, such as files, log servers and system logs. -By default, performance data is sent to files organized per system, per partition and per test as follows: - - .. code-block:: none - - {perflog_basedir}/ - system1/ - partition1/ - test_name.log - partition2/ - test_name.log - ... - system2/ - ... - -Whenever a test executes on a specific system/partition combination, the obtained performance data will be appended to a file. -A performance log record has the following format by default: - -.. code-block:: none - - 2019-10-23T13:46:05|reframe 2.20-dev2|Example7Test on daint:gpu using PrgEnv-cray|jobid=813559|num_tasks=1|perf=49.681565|ref=50.0 (l=-0.1, u=0.1)|Gflop/s - 2019-10-23T13:46:27|reframe 2.20-dev2|Example7Test on daint:gpu using PrgEnv-gnu|jobid=813560|num_tasks=1|perf=50.737651|ref=50.0 (l=-0.1, u=0.1)|Gflop/s - 2019-10-23T13:46:48|reframe 2.20-dev2|Example7Test on daint:gpu using PrgEnv-pgi|jobid=813561|num_tasks=1|perf=50.720164|ref=50.0 (l=-0.1, u=0.1)|Gflop/s - -There is information about the actual test run and the performance data obtained for the different performance variables. -The log record format is fully configurable as well as the directory structure. -More information can be found in the `configuration reference `__. - - -More Advanced Performance Logging -================================= - -ReFrame offers additional capabilities for performance logging. -You can instruct it to send data to a log management server, which provides a dedicated service for handling large amount of logs. -These services are usually combined with log analysis tools, such `Grafana `__ and `Kibana `__, which offer specialized functionality for log analysis. -If your site uses a centralized log management, you could feed the ReFrame performance logs, create dashboards and correlate it with other events on the system. - -There are two ways to achieve this in ReFrame, either through the `graylog `__ log handler or the `syslog `__ log handler. -The first one sends data to a `Graylog `__ server, whereas the second one sends data to a Syslog server. -A usual setup for centers is to feed all the syslog data to a centralized log management system, where the logs can be visualized and analyzed. - -The following image is a snapshot of an actual dashboard visualizing `GROMACS `__ performance data from ReFrame over time. - - -.. figure:: _static/img/gromacs-perf.png - :align: center - :alt: Visualization of GROMACS performance data from ReFrame over time. - - ------------------------------ -Adjusting verbosity of output ------------------------------ - -As discussed in the `configuration guide `__, ReFrame's output is handled also by its logging mechanism. -Increasing or decreasing the verbosity of its output can be achieved by adjusting the level of the log handler that sends data to standard output. -Alternatively, you may increase the verbosity level from the command line by chaining the ``-v`` or ``--verbose`` option. -Every time ``-v`` is specified, the next verbosity level will be selected for the output. -For example, if the initial level of the output log handler is set to ``info`` (in the configuration file), specifying ``-v`` twice will make ReFrame spit out all ``debug`` messages. - -.. note:: - - .. versionadded:: 2.16 - - The ``--verbose`` option was added. - - - -Asynchronous Execution of Regression Checks -------------------------------------------- - -From version `2.4 `__, ReFrame supports asynchronous execution of regression tests. -This execution policy is the default one. -To enforce a sequential execution of the regression tests the ``serial`` execution policy can be enabled by passing the option ``--exec-policy=serial`` to the command line. -The asynchronous execution policy parallelizes only the `running phase `__ of the tests. -The rest of the phases remain sequential. - -A limit of concurrent jobs (pending and running) may be `configured `__ for each virtual system partition. -As soon as the concurrency limit of a partition is reached, ReFrame will hold the execution of new regression tests until a slot is released in that partition. - -When executing in asynchronous mode, ReFrame's output differs from the sequential execution. -The final result of the tests will be printed at the end and additional messages may be printed to indicate that a test is held. -Here is an example output of ReFrame using asynchronous execution policy: - -.. code-block:: none - - Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/ --exec-policy=async -r - Reframe version: 3.0-dev3 (rev: 0c62d00c) - Launched by user: USER - Launched on host: daint105 - Reframe paths - ============= - Check prefix : - Check search path : '/path/to/reframe/tutorial' - Current working dir : /path/to/reframe - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Perf. logging prefix : /path/to/reframe/perflogs - [==========] Running 13 check(s) - [==========] Started on Wed Mar 25 10:55:35 2020 - - [----------] started processing Example1Test (Simple matrix-vector multiplication example) - [ RUN ] Example1Test on daint:login using PrgEnv-cray - [ RUN ] Example1Test on daint:login using PrgEnv-gnu - [ RUN ] Example1Test on daint:login using PrgEnv-intel - [ RUN ] Example1Test on daint:login using PrgEnv-pgi - [ RUN ] Example1Test on daint:gpu using PrgEnv-cray - [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu - [ RUN ] Example1Test on daint:gpu using PrgEnv-intel - [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi - [ RUN ] Example1Test on daint:mc using PrgEnv-cray - [ RUN ] Example1Test on daint:mc using PrgEnv-gnu - [ RUN ] Example1Test on daint:mc using PrgEnv-intel - [ RUN ] Example1Test on daint:mc using PrgEnv-pgi - [----------] finished processing Example1Test (Simple matrix-vector multiplication example) - - [----------] started processing Example2aTest (Matrix-vector multiplication example with OpenMP) - [ RUN ] Example2aTest on daint:login using PrgEnv-cray - [ RUN ] Example2aTest on daint:login using PrgEnv-gnu - [ RUN ] Example2aTest on daint:login using PrgEnv-intel - [ RUN ] Example2aTest on daint:login using PrgEnv-pgi - [ RUN ] Example2aTest on daint:gpu using PrgEnv-cray - [ RUN ] Example2aTest on daint:gpu using PrgEnv-gnu - [ RUN ] Example2aTest on daint:gpu using PrgEnv-intel - [ RUN ] Example2aTest on daint:gpu using PrgEnv-pgi - [ RUN ] Example2aTest on daint:mc using PrgEnv-cray - [ RUN ] Example2aTest on daint:mc using PrgEnv-gnu - [ RUN ] Example2aTest on daint:mc using PrgEnv-intel - [ RUN ] Example2aTest on daint:mc using PrgEnv-pgi - [----------] finished processing Example2aTest (Matrix-vector multiplication example with OpenMP) - - [----------] waiting for spawned checks to finish - [ OK ] ( 1/101) Example1Test on daint:login using PrgEnv-pgi - [ OK ] ( 2/101) Example1Test on daint:login using PrgEnv-gnu - [ OK ] ( 3/101) OpenMPTest on daint:login using PrgEnv-pgi - [ OK ] ( 4/101) SerialTest on daint:mc using PrgEnv-gnu - [ OK ] ( 5/101) Example6Test on daint:mc using PrgEnv-gnu - [ OK ] ( 6/101) Example1Test on daint:login using PrgEnv-cray - - [ OK ] ( 84/101) SerialTest on daint:gpu using PrgEnv-cray - [ OK ] ( 85/101) MPITest on daint:gpu using PrgEnv-gnu - [ OK ] ( 86/101) OpenMPTest on daint:gpu using PrgEnv-pgi - [ OK ] ( 87/101) OpenMPTest on daint:gpu using PrgEnv-gnu - [ OK ] ( 88/101) MPITest on daint:gpu using PrgEnv-cray - [ OK ] ( 89/101) OpenMPTest on daint:gpu using PrgEnv-cray - [ OK ] ( 90/101) OpenMPTest on daint:gpu using PrgEnv-intel - [ OK ] ( 91/101) MPITest on daint:gpu using PrgEnv-intel - [ OK ] ( 92/101) OpenACCTest on daint:gpu using PrgEnv-pgi - [ OK ] ( 93/101) Example1Test on daint:gpu using PrgEnv-cray - [ OK ] ( 94/101) CudaTest on daint:gpu using PrgEnv-gnu - [ OK ] ( 95/101) MPITest on daint:gpu using PrgEnv-pgi - [ OK ] ( 96/101) CudaTest on daint:gpu using PrgEnv-pgi - [ OK ] ( 97/101) Example1Test on daint:gpu using PrgEnv-intel - [ OK ] ( 98/101) CudaTest on daint:gpu using PrgEnv-cray - [ OK ] ( 99/101) Example1Test on daint:gpu using PrgEnv-pgi - [ OK ] (100/101) OpenACCTest on daint:gpu using PrgEnv-cray - [ OK ] (101/101) Example1Test on daint:gpu using PrgEnv-gnu - [----------] all spawned checks have finished - - [ PASSED ] Ran 101 test case(s) from 13 check(s) (0 failure(s)) - [==========] Finished on Wed Mar 25 13:40:48 2020 - - -The asynchronous execution policy may provide significant overall performance benefits for run-only regression tests. -For compile-only and normal tests that require a compilation, the execution time will be bound by the total compilation time of the test. - -.. note:: - .. versionchanged:: 3.0 - - The asynchronous execution policy has become the default. - - -Manipulating modules --------------------- - -.. versionadded:: 2.11 - -.. note:: - .. versionchanged:: 2.19 - Module self loops are now allowed in *module mappings*. - -ReFrame allows you to change the modules loaded by a regression test on-the-fly without having to edit the regression test file. -This feature is extremely useful when you need to quickly test a newer version of a module, but it also allows you to completely decouple the module names used in your regression tests from the real module names in a system, thus making your test even more portable. -This is achieved by defining *module mappings*. - -There are two ways to pass module mappings to ReFrame. -The first is to use the ``--map-module`` command-line option, which accepts a module mapping. -For example, the following line maps the module ``test_module`` to the module ``real_module``: - -.. code-block:: none - - --map-module='test_module: real_module' - -In this case, whenever ReFrame is asked to load ``test_module``, it will load ``real_module``. -Any string without spaces may be accepted in place of ``test_module`` and ``real_module``. -You can also define multiple module mappings at once by repeating the ``--map-module``. -If more than one mapping is specified for the same module, then the last mapping will take precedence. -It is also possible to map a single module to more than one target. -This can be done by listing the target modules separated by spaces in the order that they should be loaded. -In the following example, ReFrame will load ``real_module0`` and ``real_module1`` whenever the ``test_module`` is encountered: - -.. code-block:: none - - --map-module 'test_module: real_module0 real_module1' - -The second way of defining mappings is by listing them on a file, which you can then pass to ReFrame through the command-line option ``--module-mappings``. -Each line on the file corresponds to the definition of a mapping for a single module. -The syntax of the individual mappings in the file is the same as with the option ``--map-module`` and the same rules apply regarding repeated definitions. -Text starting with ``#`` is considered a comment and is ignored until the end of line is encountered. -Empty lines are ignored. -The following block shows an example of module mapping file: - -.. code-block:: none - - module-1: module-1a # an inline comment - module-2: module-2a module-2b module-2c - - # This is a full line comment - module-4: module-4a module-4b - -If both ``--map-module`` and ``--module-mappings`` are passed, ReFrame will first create a mapping from the definitions on the file and it will then process the definitions passed with the ``--map-module`` options. -As usual, later definitions will override the former. - -A final note on module mappings. -Module mappings can be arbitrarily deep as long as they do not form a cycle. -In this case, ReFrame will issue an error (denoting the offending cyclic dependency). -For example, suppose having the following mapping file: - -.. code-block:: none - - cudatoolkit: foo - foo: bar - bar: foobar - foobar: cudatoolkit - -If you now try to run a test that loads the module `cudatoolkit`, the following error will be yielded: - -.. code-block:: none - - ------------------------------------------------------------------------------ - FAILURE INFO for Example7Test - * System partition: daint:gpu - * Environment: PrgEnv-gnu - * Stage directory: None - * Job type: batch job (id=-1) - * Maintainers: ['you-can-type-your-email-here'] - * Failing phase: setup - * Reason: caught framework exception: module cyclic dependency: cudatoolkit->foo->bar->foobar->cudatoolkit - ------------------------------------------------------------------------------ - -On the other hand, module mappings containing self loops are allowed. -In the following example, ReFrame will load both ``module-1`` and ``module-2`` whenever the ``module-1`` is encountered: - -.. code-block:: none - - --map-module 'module-1: module-1 module-2' - -Controlling the Flexible Node Allocation ----------------------------------------- - -.. versionadded:: 2.15 - -.. note:: - .. versionchanged:: 2.21 - Flexible task allocation is now based on number of nodes. - -.. warning:: - The command line option ``--flex-alloc-tasks`` is now deprecated, you should use ``--flex-alloc-nodes`` instead. - - -ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks ` attribute is set to a value ``<=0``. -By default, ReFrame will spawn such a test on all the idle nodes of the current system partition. -This behavior can be adjusted using the ``--flex-alloc-nodes`` command line option. -This option accepts three values: - - 1. ``idle``: (default) In this case, ReFrame will set the number of tasks to the number of idle nodes of the current logical partition multiplied by the :attr:`num_tasks_per_node ` attribute of the particular test. - 2. ``all``: In this case, ReFrame will set the number of tasks to the number of all the nodes of the current logical partition multiplied by the :attr:`num_tasks_per_node ` attribute of the particular test. - - 3. Any positive integer: In this case, ReFrame will set the number of tasks to the given value multiplied by the :attr:`num_tasks_per_node ` attribute of the particular test. - -The flexible allocation of number of nodes takes into account any additional logical constraint imposed by the command line options affecting the job allocation, such as ``--partition``, ``--reservation``, ``--nodelist``, ``--exclude-nodes`` and ``--job-option`` (if the scheduler option passed to the latter imposes a restriction). -Notice that ReFrame will issue an error if the resulting number of nodes is zero. - -For example, using the following options would run a flexible test on all the nodes of reservation ``foo`` except the nodes ``n0[1-5]``: - -.. code-block:: bash - - --flex-alloc-nodes=all --reservation=foo --exclude-nodes=n0[1-5] - - -.. note:: - Flexible node allocation is supported only for the Slurm scheduler backend. - -.. warning:: - Test cases resulting from flexible ReFrame tests may not be run using the asynchronous execution policy, because the nodes satisfying the required criteria will be allocated for the first test case, causing all subsequent ones to fail. - -Testing non-default Cray Programming Environments -------------------------------------------------- - -.. versionadded:: 2.20 - - -Cray machines provide a set of compilers, scientific software libraries and an MPI implementation that is optimized for the Cray hardware. -These comprise the Cray Programming Environment (PE). -All the functionality of the PE is structured around the default versions (or modules) of the libraries. -If a non-default library or a non-default PE are to be tested, users have to do the following after having loaded all of their Cray modules: - -.. code:: bash - - export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH - - -In order to test a non-default Cray Programming Environment with ReFrame you have to pass the ``--non-default-craype``. -This will cause ReFrame to export ``LD_LIBRARY_PATH`` as shown above. -Here is an example that shows how to test a non-default Cray PE with ReFrame: - -.. code:: bash - - module load cdt/19.08 - reframe --non-default-craype -r - - -Since CDT 19.11 you can load the CDT module from within ReFrame as follows: - -.. code:: bash - - reframe -m cdt/20.03 --non-default-craype -r diff --git a/docs/sanity_functions_reference.rst b/docs/sanity_functions_reference.rst index a92b653304..05ef3e37a1 100644 --- a/docs/sanity_functions_reference.rst +++ b/docs/sanity_functions_reference.rst @@ -1,5 +1,70 @@ +========================== Sanity Functions Reference --------------------------- +========================== + +*Sanity functions* are the functions used with the :attr:`sanity_patterns ` and :attr:`perf_patterns `. +The key characteristic of these functions is that they are not executed the time they are called. +Instead they are evaluated at a later point by the framework (inside the :func:`check_sanity ` and :func:`check_performance ` methods). +Any sanity function may be evaluated either explicitly or implicitly. + + +Explicit evaluation of sanity functions +--------------------------------------- + +Sanity functions may be evaluated at any time by calling :func:`evaluate` on their return value or by passing the result of a sanity function to the :func:`reframe.utility.sanity.evaluate()` free function. + + +Implicit evaluation of sanity functions +--------------------------------------- + +Sanity functions may also be evaluated implicitly in the following situations: + +- When you try to get their truthy value by either explicitly or implicitly calling :func:`bool ` on their return value. + This implies that when you include the result of a sanity function in an :keyword:`if` statement or when you apply the :keyword:`and`, :keyword:`or` or :keyword:`not` operators, this will trigger their immediate evaluation. +- When you try to iterate over their result. + This implies that including the result of a sanity function in a :keyword:`for` statement will trigger its evaluation immediately. +- When you try to explicitly or implicitly get its string representation by calling :func:`str ` on its result. + This implies that printing the return value of a sanity function will automatically trigger its evaluation. + + +Categories of sanity functions +------------------------------ + +Currently ReFrame provides three broad categories of sanity functions: + +1. Deferrable replacements of certain Python built-in functions. + These functions simply delegate their execution to the actual built-ins. +2. Assertion functions. + These functions are used to assert certain conditions and they either return :class:`True` or raise :class:`reframe.core.exceptions.SanityError` with a message describing the error. + Users may provide their own formatted messages through the ``msg`` argument. + For example, in the following call to :func:`assert_eq` the ``{0}`` and ``{1}`` placeholders will obtain the actual arguments passed to the assertion function. + + .. code:: python + + assert_eq(a, 1, msg="{0} is not equal to {1}") + + If in the user provided message more placeholders are used than the arguments of the assert function (except the ``msg`` argument), no argument substitution will be performed in the user message. +3. Utility functions. + The are functions that you will normally use when defining :attr:`sanity_patterns ` and :attr:`perf_patterns `. + They include, but are not limited to, functions to iterate over regex matches in a file, extracting and converting values from regex matches, computing statistical information on series of data etc. + + +Users can write their own sanity functions as well. +The page ":doc:`deferrables`" explains in detail how sanity functions work and how users can write their own. + + +.. py:decorator:: sanity_function + + Sanity function decorator. + + The evaluation of the decorated function will be deferred and it will become suitable for use in the sanity and performance patterns of a regression test. + + .. code:: python + + @sanity_function + def myfunc(*args): + do_sth() + .. automodule:: reframe.utility.sanity :members: diff --git a/docs/started.rst b/docs/started.rst index ad5d1b7572..96625580d1 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -5,38 +5,79 @@ Getting Started Requirements ------------ -* Python 3.6 or higher. Python 2 is not supported. +* Python 3.6 or higher. + Python 2 is not supported. +* Required Python packages can be found in the ``requirements.txt`` file, which you can install as follows: - .. note:: - .. versionchanged:: 2.8 - A functional TCL modules system is no more required. ReFrame can now operate without a modules system at all. + .. code:: bash - .. note:: - .. versionchanged:: 3.0 - Support for Python 3.5 has been dropped. + pip3 install -r requirements.txt -Optional -~~~~~~~~ -* For running the unit tests of the framework, the `pytest `__ unittesting framework is needed. +--------------------- +Optional Requirements +--------------------- + +If you want to run the framework's unit tests, you will also need a C compiler that is able to compile a "Hello, World!" program and recognize the ``-O3`` option. + + +.. note:: + .. versionchanged:: 2.8 + + A functional TCL modules system is no more required. ReFrame can now operate without a modules system at all. + +.. note:: + .. versionchanged:: 3.0 + + Support for Python 3.5 has been dropped. + -You are advised to run the `unit tests <#running-the-unit-tests>`__ of the framework after installing it on a new system to make sure that everything works fine. Getting the Framework --------------------- -To get the latest stable version of the framework, you can just clone it from the `github `__ project page: +ReFrame's latest stable version is available through different channels: + +- As a `PyPI `__ package: + + .. code:: bash + + pip install reframe-hpc + +- As a `Spack `__ package: + + .. code:: bash + + spack install reframe + + +- As an `EasyBuild `__ package: + + .. code:: bash + + eb easybuild/easyconfigs/r/ReFrame/ReFrame-VERSION.eb -r + + + +------------------------------- +Getting the Latest and Greatest +------------------------------- + +If you want the latest development version or any pre-release, you can clone ReFrame from Github: .. code:: bash git clone https://github.com/eth-cscs/reframe.git -Alternatively, you can pick a previous stable version by downloading it from the previous `releases `__ section. + +Pre-release versions are denoted with the ``devX`` suffix and are `tagged `__ in the repository. + Running the Unit Tests ---------------------- -After you have downloaded the framework, it is important to run the unit tests of to make sure that everything is set up correctly: +You can optionally run the framework's unit tests to make sure that everything is set up correctly: + .. code:: bash @@ -46,43 +87,87 @@ The output should look like the following: .. code:: bash - collected 442 items - - unittests/test_argparser.py .. [ 0%] - unittests/test_cli.py ....s........... [ 4%] - unittests/test_config.py ............... [ 7%] - unittests/test_deferrable.py .............................................. [ 17%] - unittests/test_environments.py sss...s..... [ 20%] - unittests/test_exceptions.py ............. [ 23%] - unittests/test_fields.py .................... [ 28%] - unittests/test_launchers.py .............. [ 31%] - unittests/test_loader.py ......... [ 33%] - unittests/test_logging.py ..................... [ 38%] - unittests/test_modules.py ........ssssssssssssssss............................ [ 49%] - unittests/test_pipeline.py ....s..s......................... [ 57%] - unittests/test_policies.py ............................... [ 64%] - unittests/test_runtime.py . [ 64%] - unittests/test_sanity_functions.py ............................................... [ 75%] - .............. [ 78%] - unittests/test_schedulers.py ..........s.s......ss...................s.s......ss. [ 90%] - unittests/test_script_builders.py . [ 90%] - unittests/test_utility.py ......................................... [ 99%] - unittests/test_versioning.py .. [100%] - - ======================== 411 passed, 31 skipped in 28.10 seconds ========================= - -You will notice in the output that all the job submission related tests have been skipped. -The test suite detects if the current system has a job submission system and is configured for ReFrame (see `Configuring ReFrame for your site `__) and it will skip all the unsupported unit tests. -As soon as you configure ReFrame for your system, you can rerun the test suite to check that job submission unit tests pass as well. -Note here that some unit tests may still be skipped depending on the configured job submission system. + ======================================== test session starts ========================================= + platform darwin -- Python 3.7.3, pytest-4.3.0, py-1.8.0, pluggy-0.9.0 -- /usr/local/opt/python/bin/python3.7 + cachedir: .pytest_cache + rootdir: /Users/karakasv/Repositories/reframe, inifile: + collected 697 items + + unittests/test_argparser.py::test_arguments PASSED [ 0%] + unittests/test_argparser.py::test_parsing PASSED [ 0%] + unittests/test_argparser.py::test_option_precedence PASSED [ 0%] + unittests/test_argparser.py::test_option_with_config PASSED [ 0%] + unittests/test_argparser.py::test_option_envvar_conversion_error PASSED [ 0%] + unittests/test_buildsystems.py::TestMake::test_emit_from_buildsystem PASSED [ 0%] + unittests/test_buildsystems.py::TestMake::test_emit_from_env PASSED [ 1%] + unittests/test_buildsystems.py::TestMake::test_emit_no_env_defaults PASSED [ 1%] + unittests/test_buildsystems.py::TestCMake::test_emit_from_buildsystem PASSED [ 1%] + unittests/test_buildsystems.py::TestCMake::test_emit_from_env PASSED [ 1%] + unittests/test_buildsystems.py::TestCMake::test_emit_no_env_defaults PASSED [ 1%] + unittests/test_buildsystems.py::TestAutotools::test_emit_from_buildsystem PASSED [ 1%] + unittests/test_buildsystems.py::TestAutotools::test_emit_from_env PASSED [ 1%] + unittests/test_buildsystems.py::TestAutotools::test_emit_no_env_defaults PASSED [ 2%] + unittests/test_buildsystems.py::TestSingleSource::test_emit_from_env PASSED [ 2%] + unittests/test_buildsystems.py::TestSingleSource::test_emit_no_env PASSED [ 2%] + unittests/test_check_filters.py::TestCheckFilters::test_have_cpu_only PASSED [ 2%] + unittests/test_check_filters.py::TestCheckFilters::test_have_gpu_only PASSED [ 2%] + unittests/test_check_filters.py::TestCheckFilters::test_have_name PASSED [ 2%] + unittests/test_check_filters.py::TestCheckFilters::test_have_not_name PASSED [ 2%] + unittests/test_check_filters.py::TestCheckFilters::test_have_prgenv PASSED [ 3%] + unittests/test_check_filters.py::TestCheckFilters::test_have_tags PASSED [ 3%] + unittests/test_check_filters.py::TestCheckFilters::test_invalid_regex PASSED [ 3%] + unittests/test_check_filters.py::TestCheckFilters::test_partition PASSED [ 3%] + unittests/test_cli.py::test_check_success PASSED [ 3%] + unittests/test_cli.py::test_check_submit_success SKIPPED [ 3%] + unittests/test_cli.py::test_check_failure PASSED [ 3%] + <... output omitted ...> + unittests/test_utility.py::TestPpretty::test_simple_types PASSED [ 95%] + unittests/test_utility.py::TestPpretty::test_mixed_types PASSED [ 95%] + unittests/test_utility.py::TestPpretty::test_obj_print PASSED [ 95%] + unittests/test_utility.py::TestChangeDirCtxManager::test_change_dir_working PASSED [ 95%] + unittests/test_utility.py::TestChangeDirCtxManager::test_exception_propagation PASSED [ 95%] + unittests/test_utility.py::TestMiscUtilities::test_allx PASSED [ 95%] + unittests/test_utility.py::TestMiscUtilities::test_decamelize PASSED [ 96%] + unittests/test_utility.py::TestMiscUtilities::test_sanitize PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_construction PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_contains PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_delitem PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_iter_items PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_iter_keys PASSED [ 96%] + unittests/test_utility.py::TestScopedDict::test_iter_values PASSED [ 97%] + unittests/test_utility.py::TestScopedDict::test_key_resolution PASSED [ 97%] + unittests/test_utility.py::TestScopedDict::test_scope_key_name_pseudoconflict PASSED [ 97%] + unittests/test_utility.py::TestScopedDict::test_setitem PASSED [ 97%] + unittests/test_utility.py::TestScopedDict::test_update PASSED [ 97%] + unittests/test_utility.py::TestReadOnlyViews::test_mapping PASSED [ 97%] + unittests/test_utility.py::TestReadOnlyViews::test_sequence PASSED [ 97%] + unittests/test_utility.py::TestOrderedSet::test_concat_files PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_construction PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_construction_empty PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_construction_error PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_difference PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_intersection PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_operators PASSED [ 98%] + unittests/test_utility.py::TestOrderedSet::test_reversed PASSED [ 99%] + unittests/test_utility.py::TestOrderedSet::test_str PASSED [ 99%] + unittests/test_utility.py::TestOrderedSet::test_union PASSED [ 99%] + unittests/test_utility.py::TestOrderedSet::test_unique_abs_paths PASSED [ 99%] + unittests/test_versioning.py::TestVersioning::test_comparing_versions PASSED [ 99%] + unittests/test_versioning.py::TestVersioning::test_version_format PASSED [ 99%] + unittests/test_versioning.py::TestVersioning::test_version_validation PASSED [100%] + + ============================== 620 passed, 77 skipped in 64.58 seconds =============================== + + +You will notice that several tests will be skipped. +ReFrame uses a generic configuration by default, so that it can run on any system. +As a result, all tests for scheduler backends, environment modules, container platforms etc. will be skipped. +As soon as you configure ReFrame specifically for your system, you may rerun the test suite using your system configuration file by passing the ``--rfm-user-config=CONFIG_FILE``. + Where to Go from Here --------------------- -The next step from here is to setup and configure ReFrame for your site, so that ReFrame can automatically recognize it and submit jobs. -Please refer to the `"Configuring ReFrame For Your Site" `__ section on how to do that. - -Before starting implementing a regression test, you should go through the `"The Regression Test Pipeline" `__ section, so as to understand the mechanism that ReFrame uses to run the regression tests. -This section will let you follow easily the `"ReFrame Tutorial" `__ as well as understand the more advanced examples in the `"Customizing Further A Regression Test" `__ section. - -To learn how to invoke the ReFrame command-line interface for running your tests, please refer to the `"Running ReFrame" `__ section. +The :doc:`configure` page guides you through the basic configuration aspects of ReFrame. +The :doc:`tutorials` will allow you to get a first idea on how to write and run ReFrame tests. +:doc:`topics` explain different aspects of the framework whereas the :doc:`manuals` provide complete reference guides for the command line interface, the configuration parameters and the programming APIs for writing tests. diff --git a/docs/topics.rst b/docs/topics.rst new file mode 100644 index 0000000000..a660f4fc2a --- /dev/null +++ b/docs/topics.rst @@ -0,0 +1,11 @@ +=============== +Advanced Topics +=============== + + +.. toctree:: + :maxdepth: 2 + + pipeline + dependencies + deferrables diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst new file mode 100644 index 0000000000..43bc01d897 --- /dev/null +++ b/docs/tutorial_advanced.rst @@ -0,0 +1,510 @@ +.. |tutorialdir_adv| replace:: :obj:`tutorial/advanced/` +.. _tutorialdir_adv: https://github.com/eth-cscs/reframe/tree/master/tutorial/advanced +.. |tutorial_adv_example1| replace:: :obj:`tutorial/advanced/src/advanced_example1.c` +.. _tutorial_adv_example1: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/advanced_example1.c +.. |limits.sh| replace:: :obj:`scripts/limits.sh` +.. _limits.sh: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/scripts/limits.sh + + + +================================================= +Tutorial 2: Customizing Further a Regression Test +================================================= + +In this section, we are going to show some more elaborate use cases of ReFrame. +The corresponding scripts as well as the source code of the examples discussed here can be found in the directory |tutorialdir_adv|_. + + +Working with Makefiles +---------------------- + +We have already shown how you can compile a single source file associated with your regression test. +In this example, we show how ReFrame can leverage Makefiles to build executables. + +Compiling a regression test through a Makefile is straightforward with ReFrame. +If the :attr:`sourcepath ` attribute refers to a directory, then ReFrame will try to figure out the type of project and select the correct build system. +If it is not a CMake or Autotools based project, it will try to use ``make`` for building it. +More specifically, ReFrame first copies the :attr:`sourcesdir ` to the stage directory at the beginning of the compilation phase, then switches to the ``{stagedir}/{sourcepath}/`` and, finally, invokes ``make``. + +.. note:: + The :attr:`sourcepath ` attribute must be a relative path refering to a subdirectory of :attr:`sourcesdir `, i.e., relative paths starting with ``..`` will be rejected. + +By default, :attr:`sourcepath ` is the empty string and :attr:`sourcesdir ` is set to ``'src/'``, if such a directory exists. +As a result, by not specifying a :attr:`sourcepath ` at all, ReFrame will eventually compile the files found in the ``src/`` directory. +This is exactly what our first example here does. + +For completeness, here are the contents of ``Makefile`` provided: + +.. literalinclude:: ../tutorial/advanced/src/Makefile + :language: makefile + +The corresponding |tutorial_adv_example1|_ source file consists of a simple printing of a message, whose content depends on the preprocessor variable ``MESSAGE``: + +.. literalinclude:: ../tutorial/advanced/src/advanced_example1.c + :language: c + +The purpose of the regression test in this case is to set the preprocessor variable ``MESSAGE`` via ``CPPFLAGS`` and then check the standard output for the message ``SUCCESS``, which indicates that the preprocessor flag has been passed and processed correctly by the Makefile. + +The contents of this regression test are the following: + +.. literalinclude:: ../tutorial/advanced/advanced_example1.py + +The important bit here is how we set up the build system for this test: + +.. literalinclude:: ../tutorial/advanced/advanced_example1.py + :lines: 18-19 + :dedent: 4 + + +First, we set the build system to :attr:`Make ` and then set the preprocessor flags for the compilation. +ReFrame will invoke ``make`` as follows: + +.. code:: + + make -j 1 CC='cc' CXX='CC' FC='ftn' NVCC='nvcc' CPPFLAGS='-DMESSAGE' + +The compiler variables (``CC``, ``CXX`` etc.) are set based on the corresponding values specified in the `coniguration `__ of the current environment. +You may instruct the build system to ignore the default values from the environment by setting the following: + +.. code-block:: python + + self.build_system.flags_from_environ = False + +In this case, ``make`` will be invoked as follows: + +.. code:: + + make -j 1 CPPFLAGS='-DMESSAGE' + +Notice that the ``-j 1`` option is always generated. +You may change the maximum build concurrency as follows: + +.. code-block:: python + + self.build_system.max_concurrency = 4 + +By setting :attr:`max_concurrency ` to :class:`None`, no limit for concurrent parallel jobs will be placed. +This means that ``make -j`` will be used for building. + +Finally, you may also customize the name of the ``Makefile``. +You can achieve that by setting the corresponding variable of the :class:`Make ` build system: + +.. code-block:: python + + self.build_system.makefile = 'Makefile_custom' + + +More details on ReFrame's build systems, you may find `here `__. + + +Retrieving the source code from a Git repository +================================================ + +It might be the case that a regression test needs to clone its source code from a remote repository. +This can be achieved in two ways with ReFrame. +One way is to set the :attr:`sourcesdir` attribute to :class:`None` and explicitly clone or checkout a repository using the :attr:`prebuild_cmd `: + +.. code-block:: python + + self.sourcesdir = None + self.prebuild_cmd = ['git clone https://github.com/me/myrepo .'] + + +By setting :attr:`sourcesdir` to :class:`None`, you are telling ReFrame that you are going to provide the source files in the stage directory. +The working directory of the :attr:`prebuild_cmd` and :attr:`postbuild_cmd` commands will be the stage directory of the test. + + +An alternative way to retrieve specifically a Git repository is to assign its URL directly to the :attr:`sourcesdir` attribute: + +.. code-block:: python + + self.sourcesdir = 'https://github.com/me/myrepo' + +ReFrame will attempt to clone this repository inside the stage directory by executing ``git clone .`` and will then procede with the compilation as described above. + + +.. note:: + ReFrame recognizes only URLs in the :attr:`sourcesdir` attribute and requires passwordless access to the repository. + This means that the SCP-style repository specification will not be accepted. + You will have to specify it as URL using the ``ssh://`` protocol (see `Git documentation page `__). + + +Add a configuration step before compiling the code +================================================== + +It is often the case that a configuration step is needed before compiling a code with ``make``. +To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make" style of build systems. +It supports `CMake-based `__ projects through the :class:`CMake ` build system, as well as `Autotools-based `__ projects through the :class:`Autotools ` build system. + +For other build systems, you can achieve the same effect using the :class:`Make ` build system and the :attr:`prebuild_cmd ` for performing the configuration step. +The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``: + +.. code-block:: python + + self.prebuild_cmd = ['./custom_configure -with-mylib'] + self.build_system = 'Make' + self.build_system.cppflags = ['-DHAVE_FOO'] + self.build_system.flags_from_environ = False + +The generated build script then will have the following lines: + +.. code-block:: bash + + ./custom_configure -with-mylib + make -j 1 CPPFLAGS='-DHAVE_FOO' + + +Implementing a Run-Only Regression Test +--------------------------------------- + +There are cases when it is desirable to perform regression testing for an already built executable. +The following test uses the ``echo`` Bash shell command to print a random integer between specific lower and upper bounds. +Here is the full regression test: + +.. literalinclude:: ../tutorial/advanced/advanced_example2.py + +There is nothing special for this test compared to those presented `earlier `__ except that it derives from the :class:`RunOnlyRegressionTest ` and that it does not contain any resources (``self.sourcesdir = None``). +Note that run-only regression tests may also have resources, as for instance a precompiled executable or some input data. The copying of these resources to the stage directory is performed at the beginning of the run phase. +For standard regression tests, this happens at the beginning of the compilation phase, instead. +Furthermore, in this particular test the :attr:`executable ` consists only of standard Bash shell commands. +For this reason, we can set :attr:`sourcesdir ` to :class:`None` informing ReFrame that the test does not have any resources. + +.. note:: + .. versionchanged:: 3.0 + It is no more necessary to explicitly set :attr:`sourcesdir ` to :class:`None` for run-only tests without resources. + + +Implementing a Compile-Only Regression Test +------------------------------------------- + +ReFrame provides the option to write compile-only tests which consist only of a compilation phase without a specified executable. +This kind of tests must derive from the :class:`CompileOnlyRegressionTest ` class provided by the framework. +The following example reuses the code of our first example in this section and checks that no warnings are issued by the compiler: + +.. literalinclude:: ../tutorial/advanced/advanced_example3.py + +The important thing to note here is that the standard output and standard error of the tests, accessible through the :attr:`stdout ` and :attr:`stderr ` attributes, are now the corresponding those of the compilation command. +So sanity checking can be done in exactly the same way as with a normal test. + +Leveraging Environment Variables +-------------------------------- + +We have already demonstrated in ":doc:`tutorial_basic`" that ReFrame allows you to load the required modules for regression tests and also set any needed environment variables. When setting environment variables for your test through the :attr:`variables ` attribute, you can assign them values of other, already defined, environment variables using the standard notation ``$OTHER_VARIABLE`` or ``${OTHER_VARIABLE}``. +The following regression test sets the ``CUDA_HOME`` environment variable to the value of the ``CUDATOOLKIT_HOME`` and then compiles and runs a simple program: + +.. literalinclude:: ../tutorial/advanced/advanced_example4.py + +Before discussing this test in more detail, let's first have a look in the source code and the Makefile of this example: + +.. literalinclude:: ../tutorial/advanced/src/advanced_example4.c + :language: c + +This program is pretty basic, but enough to demonstrate the use of environment variables from ReFrame. +It simply compares the value of the ``CUDA_HOME`` macro with the value of the environment variable ``CUDA_HOME`` at runtime, printing ``SUCCESS`` if they are not empty and match. +The Makefile for this example compiles this source by simply setting ``CUDA_HOME`` to the value of the ``CUDA_HOME`` environment variable: + +.. literalinclude:: ../tutorial/advanced/src/Makefile_example4 + :language: makefile + +Coming back now to the ReFrame regression test, the ``CUDATOOLKIT_HOME`` environment variable is defined by the ``cudatoolkit`` module. +If you try to run the test, you will see that it will succeed, meaning that the ``CUDA_HOME`` variable was set correctly both during the compilation and the runtime. +ReFrame will generate the following instructions in the shell scripts (build and run) associated with this test: + +.. code-block:: bash + + module load cudatoolkit + export CUDA_HOME=$CUDATOOLKIT_HOME + +Finally, as already mentioned `previously <#working-with-makefiles>`__, since the name of the makefile is not one of the standard ones, it must be set explicitly in the build system: + +.. literalinclude:: ../tutorial/advanced/advanced_example4.py + :lines: 21 + :dedent: 8 + +Setting a Time Limit for Regression Tests +----------------------------------------- + +ReFrame gives you the option to limit the execution time of regression tests. +The following example demonstrates how you can achieve this by limiting the execution time of a test that tries to sleep 100 seconds: + +.. literalinclude:: ../tutorial/advanced/advanced_example5.py + +The important bit here is the following line that sets the time limit for the test to one minute: + +.. literalinclude:: ../tutorial/advanced/advanced_example5.py + :lines: 17 + :dedent: 8 + +The :attr:`time_limit ` attribute is a string in the form ``dhms`` and will impose a *runtime* limit to the associated job. +It will not kill the test if the job is pending for more time. +Time limits are implemented for all the scheduler backends. + +The sanity condition for this test verifies that associated job has been canceled due to the time limit (note that this message is SLURM-specific). + +.. literalinclude:: ../tutorial/advanced/advanced_example5.py + :lines: 20-21 + :dedent: 8 + + +.. note:: + .. versionadded:: 3.0 + The :attr:`max_pending_time ` attribute was added to force termination of a test if it is pending for too long. + + +Applying a sanity function iteratively +-------------------------------------- + +It is often the case that a common sanity pattern has to be applied many times. +In this example we will demonstrate how the above situation can be easily tackled using the sanity functions offered by ReFrame. +Specifically, we would like to execute the following shell script and check that its output is correct: + +.. literalinclude:: ../tutorial/advanced/src/random_numbers.sh + :language: bash + +The above script simply prints 100 random integers between the limits given by the variables ``LOWER`` and ``UPPER``. +In the corresponding regression test we want to check that all the random numbers printed lie between 90 and 100 ensuring that the script executed correctly. +Hence, a common sanity check has to be applied to all the printed random numbers. +In ReFrame this can achieved by the use of :func:`map() ` sanity function accepting a function and an iterable as arguments. +Through :func:`map() ` the given function will be applied to all the members of the iterable object. +Note that since :func:`map() ` is a sanity function, its execution will be deferred. +The ReFrame test is the following: + +.. literalinclude:: ../tutorial/advanced/advanced_example6.py + +First the random numbers are extracted through the :func:`extractall() ` function as follows: + +.. literalinclude:: ../tutorial/advanced/advanced_example6.py + :lines: 18-19 + :dedent: 8 + +The ``numbers`` variable is a deferred iterable, which upon evaluation will return all the extracted numbers. +In order to check that the extracted numbers lie within the specified limits, we make use of the :func:`map() ` sanity function, which will apply the :func:`assert_bounded() ` to all the elements of ``numbers``. + +There is still a small complication that needs to be addressed. +The :func:`all() ` function returns :class:`True` for empty iterables, which is not what we want. +So we must ensure that all the numbers are extracted as well. +To achieve this, we make use of :func:`count() ` to get the number of elements contained in ``numbers`` combined with :func:`assert_eq() ` to check that the number is indeed 100. +Finally, both of the above conditions have to be satisfied for the program execution to be considered successful, hence the use of the :func:`and_() ` function. +Note that the :keyword:`and` operator is not deferrable and will trigger the evaluation of any deferrable argument passed to it. +The full syntax for the :attr:`sanity_patterns` is the following: + +.. literalinclude:: ../tutorial/advanced/advanced_example6.py + :lines: 20-22 + :dedent: 8 + + +.. note:: + .. versionadded:: 2.13 + ReFrame offers also the :func:`allx() ` sanity function which, conversely to the builtin :func:`all()` function, will return :class:`False` if its iterable argument is empty. + + +Customizing the Generated Job Script +------------------------------------ + +It is often the case that you must run some commands before or after the parallel launch of your executable. +This can be easily achieved by using the :attr:`pre_run ` and :attr:`post_run ` attributes of a ReFrame test. + +The following example is a slightly modified version of the previous one. +The lower and upper limits for the random numbers are now set inside a helper shell script in |limits.sh|_ and we want also to print the word ``FINISHED`` after our executable has finished. +In order to achieve this, we need to source the helper script just before launching the executable and ``echo`` the desired message just after it finishes. +Here is the test file: + +.. literalinclude:: ../tutorial/advanced/advanced_example7.py + +Notice the use of the :attr:`pre_run` and :attr:`post_run` attributes. +These are lists of shell commands that are emitted verbatim in the job script. +The generated job script for this example is the following: + +.. code-block:: bash + + #!/bin/bash -l + #SBATCH --job-name="prerun_demo_check_daint_gpu_PrgEnv-gnu" + #SBATCH --time=0:10:0 + #SBATCH --ntasks=1 + #SBATCH --output=prerun_demo_check.out + #SBATCH --error=prerun_demo_check.err + #SBATCH --constraint=gpu + module load daint-gpu + module unload PrgEnv-cray + module load PrgEnv-gnu + source scripts/limits.sh + srun ./random_numbers.sh + echo FINISHED + +ReFrame generates the job shell script using the following pattern: + +.. code-block:: bash + + #!/bin/bash -l + {job_scheduler_preamble} + {test_environment} + {pre_run} + {parallel_launcher} {executable} {executable_opts} + {post_run} + +The ``job_scheduler_preamble`` contains the directives that control the job allocation. +The ``test_environment`` are the necessary commands for setting up the environment of the test. +This is the place where the modules and environment variables specified in :attr:`modules ` and :attr:`variables ` attributes are emitted. +Then the commands specified in :attr:`pre_run ` follow, while those specified in the :attr:`post_run ` come after the launch of the parallel job. +The parallel launch itself consists of three parts: + +#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, +#. the regression test executable as specified in the :attr:`executable ` attribute and +#. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. + +A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. + + +Working with parameterized tests +-------------------------------- + +.. versionadded:: 2.13 + +We have seen already in the `basic tutorial `__ how we could better organize the tests so as to avoid code duplication by using test class hierarchies. +An alternative technique, which could also be used in parallel with the class hierarchies, is to use `parameterized tests`. +The following is a test that takes a ``variant`` parameter, which controls which variant of the code will be used. +Depending on that value, the test is set up differently: + +.. literalinclude:: ../tutorial/advanced/advanced_example8.py + +If you have already gone through the :doc:`tutorial_basic`, this test can be easily understood. +The new bit here is the :func:`@parameterized_test ` decorator of the :class:`MatrixVectorTest` class. +This decorator takes an arbitrary number of arguments, which are either of a sequence type (i.e., list, tuple etc.) or of a mapping type (i.e., dictionary). +Each of the decorator's arguments corresponds to the constructor arguments of the decorated test that will be used to instantiate it. +In the example shown, the test will be instantiated twice, once passing ``variant`` as ``MPI`` and a second time with ``variant`` passed as ``OpenMP``. +The framework will try to generate unique names for the generated tests by stringifying the arguments passed to the test's constructor: + +.. code-block:: none + + [ReFrame Setup] + version: 3.0-dev6 (rev: 89d50861) + command: './bin/reframe -C tutorial/config/settings.py -c tutorial/advanced/advanced_example8.py -l' + launched by: karakasv@daint101 + working directory: '/path/to/reframe' + check search path: (R) '/path/to/reframe/tutorial/advanced/advanced_example8.py' + stage directory: '/path/to/reframe/stage' + output directory: '/path/to/reframe/output' + + [List of matched checks] + - MatrixVectorTest_MPI (found in /path/to/reframe/tutorial/advanced/advanced_example8.py) + - MatrixVectorTest_OpenMP (found in /path/to/reframe/tutorial/advanced/advanced_example8.py) + + Found 2 check(s). + + +There are a couple of different ways that we could have used the :func:`@parameterized_test ` decorator. +One is to use dictionaries for specifying the instantiations of our test class. +The dictionaries will be converted to keyword arguments and passed to the constructor of the test class: + +.. code-block:: python + + @rfm.parameterized_test({'variant': 'MPI'}, {'variant': 'OpenMP'}) + + +Another way, which is quite useful if you want to generate lots of different tests at the same time, is to use either `list comprehensions `__ or `generator expressions `__ for specifying the different test instantiations: + +.. code-block:: python + + @rfm.parameterized_test(*([variant] for variant in ['MPI', 'OpenMP'])) + + +.. tip:: + Combining parameterized tests and test class hierarchies can offer you a very flexible way for generating multiple related tests at once keeping at the same time the maintenance cost low. + We use this technique extensively in our tests. + + +Flexible Regression Tests +------------------------- + +.. versionadded:: 2.15 + +ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks ` attribute is set to a negative value or zero. +In ReFrame's terminology, such tests are called *flexible*. +Negative values indicate the minimum number of tasks that are acceptable for this test (a value of ``-4`` indicates that at least ``4`` tasks are required). +A zero value indicates the default minimum number of tasks which is equal to :attr:`num_tasks_per_node `. + +By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted with the |--flex-alloc-nodes|_ command-line option. +Flexible tests are very useful for diagnostics tests, e.g., tests for checking the health of a whole set nodes. +In this example, we demonstrate this feature through a simple test that runs ``hostname``. +The test will verify that all the nodes print the expected host name: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + +The first thing to notice in this test is that :attr:`num_tasks ` is set to zero. +This is a requirement for flexible tests: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + :lines: 17 + :dedent: 8 + +The sanity function of this test simply counts the host names and verifies that they are as many as expected: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + :lines: 19-22 + :dedent: 8 + +Notice, however, that the sanity check does not use :attr:`num_tasks` directly, but rather access the attribute through the :func:`sn.getattr() ` sanity function, which is a replacement for the :func:`getattr` builtin. +The reason for that is that at the time the sanity check expression is created, :attr:`num_tasks` is ``0`` and it will only be set to its actual value during the run phase. +Consequently, we need to defer the attribute retrieval, thus we use the :func:`sn.getattr() ` sanity function instead of accessing it directly + + +.. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` +.. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes + + + +Testing containerized applications +---------------------------------- + +.. versionadded:: 2.20 + + +ReFrame can be used also to test applications that run inside a container. +A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. +The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + +A container-based test in ReFrame requires that the :attr:`container_platform ` is set: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 17 + +This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. +In this case, the test will be using `Singularity `__ as a container platform. +If such a platform is not configured for the current system, the test will fail. +For a complete list of supported container platforms, the user is referred to the `configuration reference `__. + +As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 17-20 + +These two attributes are mandatory for container-based check. +The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. +It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. + +In the above example, ReFrame will run the container as follows: + +.. code:: shell + + singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' + +By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. +The user commands then are then run from that directory one after the other. +Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. + +Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 21 + +Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: + +.. code-block:: python + + self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), + ('/path/to/host/dir2', '/path/to/container/mount_point2')] + + +For a complete list of the available attributes of a specific container platform, the reader is referred to `ReFrame Programming APIs `__ guide. diff --git a/docs/tutorial.rst b/docs/tutorial_basic.rst similarity index 74% rename from docs/tutorial.rst rename to docs/tutorial_basic.rst index 9ecd0971ca..8d4dac236c 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial_basic.rst @@ -1,31 +1,36 @@ -================ -ReFrame Tutorial -================ +.. |tutorialdir| replace:: :obj:`tutorial/` +.. _tutorialdir: https://github.com/eth-cscs/reframe/tree/master/tutorial +.. |tutorial_settings| replace:: :obj:`tutorial/config/settings.py` +.. _tutorial_settings: https://github.com/eth-cscs/reframe/blob/master/tutorial/config/settings.py +.. |tutorial_matvec| replace:: :obj:`tutorial/src/example_matrix_vector_multiplication.c` +.. _tutorial_matvec: https://github.com/eth-cscs/reframe/blob/master/tutorial/src/example_matrix_vector_multiplication.c +.. |tutorial_matvec_hybrid| replace:: :obj:`tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c` +.. _tutorial_matvec_hybrid: https://github.com/eth-cscs/reframe/blob/master/tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c + + +====================== +Tutorial 1: The Basics +====================== This tutorial will guide you through writing your first regression tests with ReFrame. We will start with the most common and simple case of a regression test that compiles a code, runs it and checks its output. We will then expand this example gradually by adding functionality and more advanced sanity and performance checks. By the end of the tutorial, you should be able to start writing your first regression tests with ReFrame. -If you just want to get a quick feeling of how it is like writing a regression test in ReFrame, you can start directly from here. -However, if you want to get a better understanding of what is happening behind the scenes, we recommend to have a look also in `"The Regression Test Pipeline" `__ section. - -All the tutorial examples can be found in ``/tutorial/``. +All the tutorial examples can be found under the |tutorialdir|_ directory in ReFrame's source code. +Regardless how you have `installed `__ ReFrame, you can get the tutorial examples by cloning the `project `__ on Github. -For the configuration of the system, we provide a minimal configuration file for Piz Daint, where we have tested all the tutorial examples. -The site configuration that we used for this tutorial is the following: +This tutorial is tailored to the Piz Daint supercomputer, but you can use the tutorial tests on your system with slight adaptations. +The configuration file for the tutorial can be found in |tutorial_settings|_ and we list it also here for completeness: .. literalinclude:: ../tutorial/config/settings.py - :lines: 16-84 - :dedent: 4 + :lines: 10-137 -You can find the full ``settings.py`` file ready to be used by ReFrame in ``/tutorial/config/settings.py``. -You may first need to go over the `"Configuring ReFrame For Your Site" `__ section, in order to prepare the framework for your systems. The First Regression Test ------------------------- -The following is a simple regression test that compiles and runs a serial C program, which computes a matrix-vector product (``tutorial/src/example_matrix_multiplication.c``), and verifies its sane execution. +The following is a simple regression test that compiles and runs a serial C program, which computes a matrix-vector product (|tutorial_matvec|_), and verifies its sane execution. As a sanity check, it simply looks for a specific output in the output of the program. Here is the full code for this test: @@ -34,8 +39,8 @@ Here is the full code for this test: A regression test written in ReFrame is essentially a Python class that must eventually derive from :class:`RegressionTest `. To make a test visible to the framework, you must decorate your final test class with one of the following decorators: -* ``@simple_test``: for registering a single parameterless instantiation of your test. -* ``@parameterized_test``: for registering multiple instantiations of your test. +* :func:`@simple_test `: for registering a single parameterless instantiation of your test. +* :func:`@parameterized_test `: for registering multiple instantiations of your test. Let's see in more detail how the ``Example1Test`` is defined: @@ -50,24 +55,22 @@ In this example, the auto-generated test name is simply ``Example1Test``. You may change the name of the test later in the constructor by setting the :attr:`name ` attribute. .. note:: - Calling ``super().__init__()`` inside the constructor of a test is no more needed. - .. versionchanged:: 2.19 + Calling ``super().__init__()`` inside the constructor of a test is no more needed. .. warning:: + .. versionadded:: 2.12 ReFrame requires that the names of all the tests it loads are unique. In case of name clashes, it will refuse to load the conflicting test. - .. versionadded:: 2.12 - The next line sets a more detailed description of the test: .. literalinclude:: ../tutorial/example1.py :lines: 12 :dedent: 8 -This is optional and it defaults to the auto-generated test's name, if not specified. +This is optional and it defaults to the auto-generated name of the test, if not specified. .. note:: If you explicitly set only the name of the test, the description will not be automatically updated and will still keep its default value. @@ -80,9 +83,11 @@ The next two lines specify the systems and the programming environments that thi Both of these variables accept a list of system names or environment names, respectively. The ``*`` symbol is a wildcard meaning any system or any programming environment. -The system and environment names listed in these variables must correspond to names of systems and environments defined in the ReFrame's `settings file `__. +The system and environment names listed in these variables must correspond to names of systems and environments defined in the ReFrame's :doc:`configuration file `. + +.. note:: -.. ..note:: If a name specified in these lists does not appear in the settings file, it will be simply ignored. + If a name specified in these lists does not appear in the settings file, it will be simply ignored. When specifying system names you can always specify a partition name as well by appending ``:`` to the system's name. For example, given the configuration for our tutorial, ``daint:gpu`` would refer specifically to the ``gpu`` virtual partition of the system ``daint``. @@ -95,7 +100,7 @@ The next line specifies the source file that needs to be compiled: :dedent: 8 ReFrame expects any source files, or generally resources, of the test to be inside an ``src/`` directory, which is at the same level as the regression test file. -If you inspect the directory structure of the ``tutorial/`` folder, you will notice that: +If you inspect the directory structure of the |tutorialdir|_ folder, you will notice that: .. code-block:: none @@ -108,7 +113,7 @@ Notice also that you need not specify the programming language of the file you a ReFrame will automatically pick the correct compiler based on the extension of the source file. The exact compiler that is going to be used depends on the programming environment that the test is running with. For example, given our configuration, if it is run with ``PrgEnv-cray``, the Cray C compiler will be used, if it is run with ``PrgEnv-gnu``, the GCC compiler will be used etc. -A user can associate compilers with programming environments in the ReFrame's `settings file `__. +A user can associate compilers with programming environments in the ReFrame's :doc:`configuration file `. The next line in our first regression test specifies a list of options to be used for running the generated executable (the matrix dimension and the number of iterations in this particular example): @@ -118,7 +123,7 @@ The next line in our first regression test specifies a list of options to be use Notice that you do not need to specify the executable name. Since ReFrame compiled it and generated it, it knows the name. -We will see in the `"Customizing Further A ReFrame Regression Test" `__ section, how you can specify the name of the executable, in cases that ReFrame cannot guess its name. +We will see in ":doc:`tutorial_advanced`" how you can specify the name of the executable, in cases that ReFrame cannot guess its name. The next lines specify what should be checked for assessing the sanity of the result of the test: @@ -131,14 +136,14 @@ The :attr:`sanity_patterns `__ are special in the sense that they are evaluated lazily. You can generally treat them as normal Python functions inside a :attr:`sanity_patterns ` expression. ReFrame provides already a wide range of useful sanity functions ranging from wrappers to the standard built-in functions of Python to functions related to parsing the output of a regression test. -For a complete listing of the available functions, please have a look at the `"Sanity Functions Reference" `__. +For a complete listing of the available functions, you may have a look at the :doc:`sanity_functions_reference`. In our example, the :func:`assert_found ` function accepts a regular expression pattern to be searched in a file and either returns :class:`True` on success or raises a :class:`SanityError ` in case of failure with a descriptive message. -This function uses internally the "`re `__" module of the Python standard library, so it may accept the same `regular expression syntax `__. +This function accepts any valid `Python Regular Expression `__. As a file argument, :func:`assert_found ` accepts any filename, which will be resolved against the stage directory of the test. You can also use the :attr:`stdout ` and :attr:`stderr ` attributes to reference the standard output and standard error, respectively. -.. tip:: You need not to care about handling exceptions, and error handling in general, inside your test. +.. tip:: You don't need to care about handling exceptions, and error handling in general, inside your test. The framework will automatically abort the execution of the test, report the error and continue with the next test case. The last two lines of the regression test are optional, but serve a good role in a production environment: @@ -147,21 +152,21 @@ The last two lines of the regression test are optional, but serve a good role in :lines: 19-20 :dedent: 8 -In the :attr:`maintainers ` attribute you may store a list of people responsible for the maintenance of this test. +In the :attr:`maintainers ` attribute you may store a list of persons responsible for the maintenance of this test. In case of failure, this list will be printed in the failure summary. The :attr:`tags ` attribute is a set of tags that you can assign to this test. This is useful for categorizing the tests and helps in quickly selecting the tests of interest. -More about test selection, you can find in the `"Running ReFrame" `__ section. .. note:: The values assigned to the attributes of a :class:`RegressionTest ` are validated and if they don't have the correct type, an error will be issued by ReFrame. - For a list of all the attributes and their types, please refer to the `"Reference Guide" `__. + For a list of all the attributes and their types, please refer to the :doc:`regression_test_api` guide. + +----------------------------- Running the Tutorial Examples -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- -ReFrame offers a rich command-line interface that allows you to control several aspects of its executions. -A more detailed description can be found in the `"Running ReFrame" `__ section. +ReFrame offers a rich `command-line interface `__ that allows you to control several aspects of its executions. Here we will only show you how to run a specific tutorial test: .. code-block:: bash @@ -172,67 +177,66 @@ If everything is configured correctly for your system, you should get an output .. code-block:: none - Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r - Reframe version: 3.0-dev3 (rev: 0c62d00c) - Launched by user: USER - Launched on host: daint105 - Reframe paths - ============= - Check prefix : - Check search path : '/path/to/reframe/tutorial/example1.py' - Current working dir : /path/to/reframe - Stage dir prefix : /path/to/reframe/stage/ - Output dir prefix : /path/to/reframe/output/ - Perf. logging prefix : /path/to/reframe/perflogs - [==========] Running 1 check(s) - [==========] Started on Wed Mar 25 10:13:13 2020 - - [----------] started processing Example1Test (Simple matrix-vector multiplication example) - [ RUN ] Example1Test on daint:login using PrgEnv-cray - [ RUN ] Example1Test on daint:login using PrgEnv-gnu - [ RUN ] Example1Test on daint:login using PrgEnv-intel - [ RUN ] Example1Test on daint:login using PrgEnv-pgi - [ RUN ] Example1Test on daint:gpu using PrgEnv-cray - [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu - [ RUN ] Example1Test on daint:gpu using PrgEnv-intel - [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi - [ RUN ] Example1Test on daint:mc using PrgEnv-cray - [ RUN ] Example1Test on daint:mc using PrgEnv-gnu - [ RUN ] Example1Test on daint:mc using PrgEnv-intel - [ RUN ] Example1Test on daint:mc using PrgEnv-pgi - [----------] finished processing Example1Test (Simple matrix-vector multiplication example) - - [----------] waiting for spawned checks to finish - [ OK ] ( 1/12) Example1Test on daint:login using PrgEnv-intel - [ OK ] ( 2/12) Example1Test on daint:login using PrgEnv-cray - [ OK ] ( 3/12) Example1Test on daint:login using PrgEnv-gnu - [ OK ] ( 4/12) Example1Test on daint:login using PrgEnv-pgi - [ OK ] ( 5/12) Example1Test on daint:mc using PrgEnv-gnu - [ OK ] ( 6/12) Example1Test on daint:mc using PrgEnv-pgi - [ OK ] ( 7/12) Example1Test on daint:mc using PrgEnv-cray - [ OK ] ( 8/12) Example1Test on daint:mc using PrgEnv-intel - [ OK ] ( 9/12) Example1Test on daint:gpu using PrgEnv-intel - [ OK ] (10/12) Example1Test on daint:gpu using PrgEnv-cray - [ OK ] (11/12) Example1Test on daint:gpu using PrgEnv-gnu - [ OK ] (12/12) Example1Test on daint:gpu using PrgEnv-pgi - [----------] all spawned checks have finished - - [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Wed Mar 25 10:21:08 2020 + [ReFrame Setup] + version: 3.0-dev6 (rev: 89d50861) + command: './bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r' + launched by: user@daint101 + working directory: '/path/to/reframe' + check search path: (R) '/path/to/reframe/tutorial/example1.py' + stage directory: '/path/to/reframe/stage' + output directory: '/path/to/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Sat May 9 22:10:16 2020 + + [----------] started processing Example1Test (Simple matrix-vector multiplication example) + [ RUN ] Example1Test on daint:login using PrgEnv-cray + [ RUN ] Example1Test on daint:login using PrgEnv-gnu + [ RUN ] Example1Test on daint:login using PrgEnv-intel + [ RUN ] Example1Test on daint:login using PrgEnv-pgi + [ RUN ] Example1Test on daint:gpu using PrgEnv-cray + [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu + [ RUN ] Example1Test on daint:gpu using PrgEnv-intel + [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi + [ RUN ] Example1Test on daint:mc using PrgEnv-cray + [ RUN ] Example1Test on daint:mc using PrgEnv-gnu + [ RUN ] Example1Test on daint:mc using PrgEnv-intel + [ RUN ] Example1Test on daint:mc using PrgEnv-pgi + [----------] finished processing Example1Test (Simple matrix-vector multiplication example) + + [----------] waiting for spawned checks to finish + [ OK ] ( 1/12) Example1Test on daint:mc using PrgEnv-cray + [ OK ] ( 2/12) Example1Test on daint:gpu using PrgEnv-intel + [ OK ] ( 3/12) Example1Test on daint:gpu using PrgEnv-cray + [ OK ] ( 4/12) Example1Test on daint:login using PrgEnv-intel + [ OK ] ( 5/12) Example1Test on daint:login using PrgEnv-cray + [ OK ] ( 6/12) Example1Test on daint:mc using PrgEnv-gnu + [ OK ] ( 7/12) Example1Test on daint:gpu using PrgEnv-gnu + [ OK ] ( 8/12) Example1Test on daint:login using PrgEnv-gnu + [ OK ] ( 9/12) Example1Test on daint:login using PrgEnv-pgi + [ OK ] (10/12) Example1Test on daint:gpu using PrgEnv-pgi + [ OK ] (11/12) Example1Test on daint:mc using PrgEnv-intel + [ OK ] (12/12) Example1Test on daint:mc using PrgEnv-pgi + [----------] all spawned checks have finished + + [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Sat May 9 22:10:46 2020 + Notice how our regression test is run on every partition of the configured system and for every programming environment. Now that you have got a first understanding of how a regression test is written in ReFrame, let's try to expand our example. +-------------------------------------- Inspecting the ReFrame Generated Files -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------------- -As described in the `regression test pipeline `__ section, ReFrame generates several files during the execution of a test. +ReFrame generates several files during the execution of a test. When developing or debugging a regression test it is important to be able to locate them and inspect them. As soon as the `setup` stage of the test is executed, a stage directory specific to this test is generated. All the required resources for the test are copied to this directory, and this will be the working directory for the compilation, running, sanity and performance checking phases. -If the test is successful, this stage directory is removed, unless the ``--keep-stage-files`` option is passed in the command line. +If the test is successful, this stage directory is removed, unless the :option:`--keep-stage-files` option is passed in the command line. Before removing this directory, ReFrame copies the following files to a dedicated output directory for this test: - The generated build script and its standard output and standard error. @@ -346,7 +350,7 @@ This example introduces two new concepts: 2. We need to specify different flags for the different compilers provided by the programming environments we are testing. Notice also that we now restrict the validity of our test only to the programming environments that we know how to handle (see the :attr:`valid_prog_environs `). -To define environment variables to be set during the execution of a test, you should use the :attr:`variables ` attribute of the :class:`RegressionTest ` class. +To define environment variables to be set during the execution of a test, you should use the :attr:`variables ` attribute of the regression test. This is a dictionary, whose keys are the names of the environment variables and whose values are the values of the environment variables. Notice that both the keys and the values must be strings. @@ -364,9 +368,9 @@ Assuming our test supported only GCC, we could simply add the following lines in The :class:`SingleSource ` build system that we use here supports the compilation of a single file only. Each build system type defines a set of variables that the user can set. Based on the selected build system, ReFrame will generate a build script that will be used for building the code. -The generated build script can be found in `the stage or the output directory of the test `__, along with the output of the compilation. +The generated build script can be found in the stage or the output directory of the test, along with the output of the compilation. This way, you may reproduce exactly what ReFrame does in case of any errors. -More on the build systems feature can be found `here `__. +More on the build systems feature can be found `here `__. Getting back to our test, simply setting the ``cflags`` to ``-fopenmp`` globally in the test will make it fail for programming environments other than ``PrgEnv-gnu``, since the OpenMP flags vary for the different compilers. Ideally, we need to set the ``cflags`` differently for each programming environment. @@ -377,11 +381,10 @@ To achieve this we need to define a method that will set the compilation flags b :dedent: 4 In this function we retrieve the current environment from the :attr:`current_environ ` attribute, so we can then differentiate the build system's flags based on its name. -Note that we cannot retrieve the current programming environment inside the test's constructor, since as described in in `"The Regression Test Pipeline" `__ section, it is during the setup phase that a regression test is prepared for a new system partition and a new programming environment. +Note that we cannot retrieve the current programming environment inside the test's constructor, since as described in in the ":doc:`pipeline`" page, it is during the setup phase that a regression test is prepared for a new system partition and a new programming environment. The second important thing in this function is the ``@run_before('compile')`` decorator. This decorator will attach this function to the ``compile`` stage of the pipeline and will execute it before entering this stage. -There are six pipeline stages that are defined and can accept this type of hooks: ``setup``, ``compile``, ``run``, ``sanity``, ``performance`` and ``cleanup``. -Similarly, to the ``@run_before`` decorator, there is also the ``@run_after``, which will run the decorated function after the specified pipeline stage. +Similarly to the :func:`@run_before ` decorator, there is also the :func:`@run_after `, which will run the decorated function after the specified pipeline stage. The decorated function may have any name, but it should be a method of the test taking no arguments (i.e., its sole argument should the ``self``). You may attach multiple functions to the same stage as in the following example: @@ -406,7 +409,7 @@ You may attach multiple functions to the same stage as in the following example: In this case, the decorated functions will be executed before the compilation stage in the order that they are defined in the regression test. -There is also the possibility to attach a single function to multiple stages by stacking the ``@run_before`` or ``@run_after`` decorators. +There is also the possibility to attach a single function to multiple stages by stacking the :func:`@run_before ` or :func:`@run_after ` decorators. In the following example ``var`` will be set to ``2`` after the setup phase is executed: .. code:: python @@ -462,28 +465,29 @@ Otherwise, the base class hook will be executed. .. warning:: - Configuring your test per programming environment and per system partition by overriding the :func:`setup() ` method is deprecated. - Please refer to the `Migrate to ReFrame 3 `__ guide for more details. - .. versionchanged:: 3.0 + Configuring your test per programming environment and per system partition by overriding the :func:`setup ` method is deprecated. + Please refer to the ":doc:`migration_2_to_3`" guide for more details. -.. warning:: - Support for setting the compiler flags in the programming environment has been dropped completely. +.. warning:: .. versionchanged:: 2.17 + Support for setting the compiler flags in the programming environment has been dropped completely. + +------------------------------------------------ An alternative implementation using dictionaries -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------------------ Here we present an alternative implementation of the same test using a dictionary to hold the compilation flags for the different programming environments. The advantage of this implementation is that you move the different compilation flags in the initialization phase, where also the rest of the test's specification is, thus making it more concise. -The ``setup()`` method is now very simple: -it gets the correct compilation flags from the ``prgenv_flags`` dictionary and applies them to the build system. +The :func:`setflags` method is now very simple: +it gets the correct compilation flags from the :attr:`prgenv_flags` dictionary and applies them to the build system. .. literalinclude:: ../tutorial/example2.py - :lines: 6-9,36-62 + :lines: 6-9,41-67 .. tip:: A regression test is like any other Python class, so you can freely define your own attributes. @@ -495,7 +499,7 @@ Running on Multiple Nodes So far, all our tests run on a single node. Depending on the actual system that ReFrame is running, the test may run locally or be submitted to the system's job scheduler. In this example, we write a regression test for the MPI+OpenMP version of the matrix-vector product. -The source code of this program is in ``tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c``. +The source code of this program is in |tutorial_matvec_hybrid|_. The regression test file follows: .. literalinclude:: ../tutorial/example3.py @@ -529,19 +533,23 @@ ReFrame provides several more variables for configuring the job submission. As shown in the following Table, they follow closely the corresponding Slurm options. For schedulers that do not provide the same functionality, some of the variables may be ignored. -================================================ =========================================== - :class:`RegressionTest` attribute Corresponding SLURM option -================================================ =========================================== - ``time_limit = '10m30s`` ``--time=00:10:30`` - ``use_multithreading = True`` ``--hint=multithread`` - ``use_multithreading = False`` ``--hint=nomultithread`` - ``exclusive_access = True`` ``--exclusive`` - ``num_tasks=72`` ``--ntasks=72`` - ``num_tasks_per_node=36`` ``--ntasks-per-node=36`` - ``num_cpus_per_task=4`` ``--cpus-per-task=4`` - ``num_tasks_per_core=2`` ``--ntasks-per-core=2`` - ``num_tasks_per_socket=36`` ``--ntasks-per-socket=36`` -================================================ =========================================== +.. table:: + :align: center + + ================================= ========================== + :class:`RegressionTest` attribute Corresponding SLURM option + ================================= ========================== + ``time_limit = '10m30s`` ``--time=00:10:30`` + ``use_multithreading = True`` ``--hint=multithread`` + ``use_multithreading = False`` ``--hint=nomultithread`` + ``exclusive_access = True`` ``--exclusive`` + ``num_tasks=72`` ``--ntasks=72`` + ``num_tasks_per_node=36`` ``--ntasks-per-node=36`` + ``num_cpus_per_task=4`` ``--cpus-per-task=4`` + ``num_tasks_per_core=2`` ``--ntasks-per-core=2`` + ``num_tasks_per_socket=36`` ``--ntasks-per-socket=36`` + ================================= ========================== + Testing a GPU Code ------------------ @@ -558,10 +566,10 @@ The things to notice in this test are the restricted list of system partitions a :lines: 19 :dedent: 8 -The :attr:`modules ` variable takes a list of modules that should be loaded during the setup phase of the test. +The :attr:`modules ` variable takes a list of environment modules that should be loaded during the setup phase of the test. In this particular test, we need to load the ``craype-accel-nvidia60`` module, which enables the generation of a GPU binary from an OpenACC code. -It is also important to note that in GPU-enabled tests the number of GPUs for each node have to be specified by setting the corresponding variable :attr:`num_gpus_per_node `, as follows: +It's advisable for GPU-enabled tests to define also the :attr:`num_gpus_per_node ` attribute, since this information may be needed by some system partitions in order to get the right nodes: .. literalinclude:: ../tutorial/example4.py :lines: 20 @@ -622,10 +630,10 @@ This expression combines two conditions that need to be true, in order for the s 1. Find in standard output the same line we were looking for already in the first example. 2. Verify that the printed norm does not deviate significantly from the expected value. -The :func:`all ` function is responsible for combining the results of the individual subexpressions. -It is essentially the Python built-in `all() `__ function, exposed as a sanity function, and requires that all the elements of the iterable it takes as an argument evaluate to :class:`True`. -As mentioned before, all the ``assert_*`` functions either return :class:`True` on success or raise :class:`SanityError `. -So, if everything goes smoothly, ``sn.all()`` will evaluate to :class:`True` and sanity checking will succeed. +The :func:`all() ` function is responsible for combining the results of the individual subexpressions. +It is essentially the Python built-in :func:`all() ` function, exposed as a sanity function, and requires that all the elements of the iterable it takes as an argument evaluate to :class:`True`. +As mentioned before, all the :func:`assert_*` functions either return :class:`True` on success or raise :class:`SanityError `. +So, if everything goes smoothly, :func:`sn.all() ` will evaluate to :class:`True` and sanity checking will succeed. The expression for the second condition is more interesting. Here, we want to assert that the absolute value of the difference between the expected and the found norm are below a certain value. @@ -633,7 +641,7 @@ The important thing to mention here is that you can combine the results of sanit Remember that sanity functions are not evaluated at the time you call them. They will be evaluated later by the framework during the sanity checking phase. If you include the result of a sanity function in an expression, the evaluation of the resulting expression will also be deferred. -For a detailed description of the mechanism behind the sanity functions, please have a look at `"Understanding The Mechanism Of Sanity Functions" `__ section. +For a detailed description of the mechanism behind the sanity functions, please have a look at the ":doc:`deferrables`" page. Writing a Performance Test -------------------------- @@ -641,15 +649,15 @@ Writing a Performance Test An important aspect of regression testing is checking for performance regressions. ReFrame offers a flexible way of extracting and manipulating performance data from the program output, as well as a comprehensive way of setting performance thresholds per system and system partitions. -In this example, we extend the CUDA test presented `previously `__, so as to check also the performance of the matrix-vector multiplication. +In this example, we extend the CUDA test presented `previously <#testing-a-gpu-code>`__, so as to check also the performance of the matrix-vector multiplication. .. literalinclude:: ../tutorial/example7.py The are two new variables set in this test that basically enable the performance testing: -:attr:`perf_patterns ` +- :attr:`perf_patterns ` This variable defines which are the performance patterns we are looking for and how to extract the performance values. -:attr:`reference ` +- :attr:`reference ` This variable is a collection of reference values for different systems. Let's have a closer look at each of them: @@ -670,8 +678,7 @@ Let's go over the :attr:`reference `__ of ReFrame. +This is a special type of dictionary that we call *scoped dictionary*, because it defines scopes for its keys. In order to resolve a reference value for a performance variable, ReFrame creates the following key ``::`` and looks it up inside the :attr:`reference ` dictionary. If our example, since this test is only allowed to run on the ``daint:gpu`` partition of our system, ReFrame will look for the ``daint:gpu:perf`` reference key. The ``perf`` subkey will then be searched in the following scopes in this order: @@ -683,24 +690,20 @@ Reference values in ReFrame are specified as a four-tuple comprising the referen If no unit is relevant, then you have to insert :class:`None` explicitly. Thresholds are specified as decimal fractions of the reference value. For nonnegative reference values, the lower threshold must lie in the [-1,0], whereas the upper threshold may be any positive real number or zero. In our example, the reference value for this test on ``daint:gpu`` is 50 Gflop/s ±10%. Setting a threshold value to :class:`None` disables the threshold. -If you specify a measurement unit as well, you will be able to log it the performance logs of the test; this is handy when you are inspecting or plotting the performance values. +If you specify a measurement unit as well, you will be able to log it in the performance logs of the test; this is handy when you are inspecting or plotting the performance values. ReFrame will always add a default ``*`` entry in the ``reference`` dictionary, if it does not exist, with the reference value of ``(0, None, None, )``, where ``unit`` is derived from the unit of each respective performance variable. This is useful when using ReFrame for benchmarking purposes and you would like to run a test on an unknown system. .. note:: - Reference tuples may now optionally contain units. - .. versionadded:: 2.16 - -.. note:: - A default ``*`` entry is now always added to the reference dictionary. + Reference tuples may now optionally contain units. .. versionadded:: 2.19 + A default ``*`` entry is now always added to the reference dictionary. -.. note:: - Reference tuples now require the measurement unit. .. versionchanged:: 3.0 + Reference tuples now require the measurement unit. Combining It All Together @@ -715,7 +718,7 @@ Here is the final example code that combines all the tests discussed before: This test abstracts away the common functionality found in almost all of our tutorial tests (executable options, sanity checking, etc.) to a base class, from which all the concrete regression tests derive. Each test then redefines only the parts that are specific to it. -Notice also that only the actual tests, i.e., the derived classes, are made visible to the framework through the ``@simple_test`` decorator. +Notice also that only the actual tests, i.e., the derived classes, are made visible to the framework through the :func:`@simple_test ` decorator. Decorating the base class has no meaning, because it does not correspond to an actual test. The total line count of this refactored example is less than half of that of the individual tutorial tests. @@ -724,6 +727,6 @@ Another interesting thing to note here is the base class accepting additional ad Summary ------- -This concludes our ReFrame tutorial. +This concludes the first ReFrame tutorial. We have covered all basic aspects of writing regression tests in ReFrame and you should now be able to start experimenting by writing your first useful tests. -The `next section `__ covers further topics in customizing a regression test to your needs. +The `next tutorial `__ covers further topics in customizing a regression test to your needs. diff --git a/docs/tutorial_deps.rst b/docs/tutorial_deps.rst new file mode 100644 index 0000000000..8709d531b2 --- /dev/null +++ b/docs/tutorial_deps.rst @@ -0,0 +1,177 @@ +=============================================== +Tutorial 3: Using Dependencies in ReFrame Tests +=============================================== + +.. versionadded:: 2.21 + + +A ReFrame test may define dependencies to other tests. +An example scenario is to test different runtime configurations of a benchmark that you need to compile, or run a scaling analysis of a code. +In such cases, you don't want to rebuild your test for each runtime configuration. +You could have a build test, which all runtime tests would depend on. +This is the approach we take with the following example, that fetches, builds and runs several `OSU benchmarks `__. +We first create a basic compile-only test, that fetches the benchmarks and builds them for the different programming environments: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 92-106 + +There is nothing particular to that test, except perhaps that you can set :attr:`sourcesdir ` to ``None`` even for a test that needs to compile something. +In such a case, you should at least provide the commands that fetch the code inside the :attr:`prebuild_cmd ` attribute. + +For the next test we need to use the OSU benchmark binaries that we just built, so as to run the MPI ping-pong benchmark. +Here is the relevant part: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 12-44 + +First, since we will have multiple similar benchmarks, we move all the common functionality to the :class:`OSUBenchmarkTestBase` base class. +Again nothing new here; we are going to use two nodes for the benchmark and we set :attr:`sourcesdir ` to ``None``, since none of the benchmark tests will use any additional resources. +The new part comes in with the :class:`OSULatencyTest` test in the following line: + + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 32 + +Here we tell ReFrame that this test depends on a test named ``OSUBuildTest``. +This test may or may not be defined in the same test file; all ReFrame needs is the test name. +By default, the :func:`depends_on() ` function will create dependencies between the individual test cases of the :class:`OSULatencyTest` and the :class:`OSUBuildTest`, such that the :class:`OSULatencyTest` using ``PrgEnv-gnu`` will depend on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-gnu``, but not on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-intel``. +This behaviour can be changed, but it is covered in detail in :doc:`dependencies`. +You can create arbitrary test dependency graphs, but they need to be acyclic. +If ReFrame detects cyclic dependencies, it will refuse to execute the set of tests and will issue an error pointing out the cycle. + +A ReFrame test with dependencies will execute, i.e., enter its `setup` stage, only after `all` of its dependencies have succeeded. +If any of its dependencies fails, the current test will be marked as failure as well. + +The next step for the :class:`OSULatencyTest` is to set its executable to point to the binary produced by the :class:`OSUBuildTest`. +This is achieved with the following specially decorated function: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 37-43 + +The :func:`@require_deps ` decorator will bind the arguments passed to the decorated function to the result of the dependency that each argument names. +In this case, it binds the ``OSUBuildTest`` function argument to the result of a dependency named ``OSUBuildTest``. +In order for the binding to work correctly the function arguments must be named after the target dependencies. +However, referring to a dependency only by the test's name is not enough, since a test might be associated with multiple programming environments. +For this reason, a dependency argument is actually bound to a function that accepts as argument the name of a target programming environment. +If no arguments are passed to that function, as in this example, the current programming environment is implied, such that ``OSUBuildTest()`` is equivalent to ``OSUBuildTest(self.current_environ.name)``. +This call returns the actual test case of the dependency that has been executed. +This allows you to access any attribute from the target test, as we do in this example by accessing the target test's stage directory, which we use to construct the path of the executable. +This concludes the presentation of the :class:`OSULatencyTest` test. The :class:`OSUBandwidthTest` is completely analogous. + +The :class:`OSUAllreduceTest` shown below is similar to the other two, except that it is parameterized. +It is essentially a scalability test that is running the ``osu_allreduce`` executable created by the :class:`OSUBuildTest` for 2, 4, 8 and 16 nodes. + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 69-89 + +The full set of OSU example tests is shown below: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + +Notice that the order in which dependencies are defined in a test file is irrelevant. +In this case, we define :class:`OSUBuildTest` at the end. +ReFrame will make sure to properly sort the tests and execute them. + +Here is the output when running the OSU tests with the asynchronous execution policy: + +.. code-block:: none + + [==========] Running 7 check(s) + [==========] Started on Wed Mar 25 13:51:06 2020 + + [----------] started processing OSUBuildTest (OSU benchmarks build test) + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-intel + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUBuildTest (OSU benchmarks build test) + + [----------] started processing OSULatencyTest (OSU latency test) + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-gnu + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-intel + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-intel + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-pgi + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSULatencyTest (OSU latency test) + + [----------] started processing OSUBandwidthTest (OSU bandwidth test) + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-intel + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-intel + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUBandwidthTest (OSU bandwidth test) + + [----------] started processing OSUAllreduceTest_2 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_2 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_4 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_4 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_8 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_8 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_16 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_16 (OSU Allreduce test) + + [----------] waiting for spawned checks to finish + [ OK ] ( 1/21) OSUBuildTest on daint:gpu using PrgEnv-pgi + [ OK ] ( 2/21) OSUBuildTest on daint:gpu using PrgEnv-gnu + [ OK ] ( 3/21) OSUBuildTest on daint:gpu using PrgEnv-intel + [ OK ] ( 4/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [ OK ] ( 5/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [ OK ] ( 6/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [ OK ] ( 7/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [ OK ] ( 8/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ OK ] ( 9/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ OK ] (10/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ OK ] (11/21) OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ OK ] (12/21) OSULatencyTest on daint:gpu using PrgEnv-pgi + [ OK ] (13/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ OK ] (14/21) OSULatencyTest on daint:gpu using PrgEnv-gnu + [ OK ] (15/21) OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [ OK ] (16/21) OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ OK ] (17/21) OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ OK ] (18/21) OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ OK ] (19/21) OSULatencyTest on daint:gpu using PrgEnv-intel + [ OK ] (20/21) OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ OK ] (21/21) OSUBandwidthTest on daint:gpu using PrgEnv-intel + [----------] all spawned checks have finished + + [ PASSED ] Ran 21 test case(s) from 7 check(s) (0 failure(s)) + [==========] Finished on Wed Mar 25 14:37:53 2020 + +Before starting running the tests, ReFrame topologically sorts them based on their dependencies and schedules them for running using the selected execution policy. +With the serial execution policy, ReFrame simply executes the tests to completion as they "arrive", since the tests are already topologically sorted. +In the asynchronous execution policy, tests are spawned and not waited for. +If a test's dependencies have not yet completed, it will not start its execution and a ``DEP`` message will be printed to denote this. + +Finally, ReFrame's runtime takes care of properly cleaning up the resources of the tests respecting dependencies. +Normally when an individual test finishes successfully, its stage directory is cleaned up. +However, if other tests are depending on this one, this would be catastrophic, since most probably the dependent tests would need the outcome of this test. +ReFrame fixes that by not cleaning up the stage directory of a test until all its dependent tests have finished successfully. diff --git a/docs/tutorials.rst b/docs/tutorials.rst new file mode 100644 index 0000000000..9e02aec23e --- /dev/null +++ b/docs/tutorials.rst @@ -0,0 +1,10 @@ +================= +ReFrame Tutorials +================= + + +.. toctree:: + + tutorial_basic + tutorial_advanced + tutorial_deps diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 74b45f55e2..96c249fed1 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -19,85 +19,85 @@ class BuildSystem(abc.ABC): ''' #: The C compiler to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the compiler defined in the current programming environment will be #: used. #: #: :type: :class:`str` - #: :default: :class:`None` - cc = fields.TypedField('cc', str, type(None)) + #: :default: ``''`` + cc = fields.TypedField('cc', str) #: The C++ compiler to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the compiler defined in the current programming environment will be #: used. #: #: :type: :class:`str` - #: :default: :class:`None` - cxx = fields.TypedField('cxx', str, type(None)) + #: :default: ``''`` + cxx = fields.TypedField('cxx', str) #: The Fortran compiler to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the compiler defined in the current programming environment will be #: used. #: #: :type: :class:`str` - #: :default: :class:`None` - ftn = fields.TypedField('ftn', str, type(None)) + #: :default: ``''`` + ftn = fields.TypedField('ftn', str) #: The CUDA compiler to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the compiler defined in the current programming environment will be #: used. #: #: :type: :class:`str` - #: :default: :class:`None` - nvcc = fields.TypedField('nvcc', str, type(None)) + #: :default: ``''`` + nvcc = fields.TypedField('nvcc', str) #: The C compiler flags to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the corresponding flags defined in the current programming environment #: will be used. #: #: :type: :class:`List[str]` - #: :default: :class:`None` - cflags = fields.TypedField('cflags', typ.List[str], type(None)) + #: :default: ``[]`` + cflags = fields.TypedField('cflags', typ.List[str]) #: The preprocessor flags to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the corresponding flags defined in the current programming environment #: will be used. #: #: :type: :class:`List[str]` - #: :default: :class:`None` - cppflags = fields.TypedField('cppflags', typ.List[str], type(None)) + #: :default: ``[]`` + cppflags = fields.TypedField('cppflags', typ.List[str]) #: The C++ compiler flags to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the corresponding flags defined in the current programming environment #: will be used. #: #: :type: :class:`List[str]` - #: :default: :class:`None` - cxxflags = fields.TypedField('cxxflags', typ.List[str], type(None)) + #: :default: ``[]`` + cxxflags = fields.TypedField('cxxflags', typ.List[str]) #: The Fortran compiler flags to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the corresponding flags defined in the current programming environment #: will be used. #: #: :type: :class:`List[str]` - #: :default: :class:`None` - fflags = fields.TypedField('fflags', typ.List[str], type(None)) + #: :default: ``[]`` + fflags = fields.TypedField('fflags', typ.List[str]) #: The linker flags to be used. - #: If set to :class:`None` and :attr:`flags_from_environ` is :class:`True`, + #: If empty and :attr:`flags_from_environ` is :class:`True`, #: the corresponding flags defined in the current programming environment #: will be used. #: #: :type: :class:`List[str]` - #: :default: :class:`None` - ldflags = fields.TypedField('ldflags', typ.List[str], type(None)) + #: :default: ``[]`` + ldflags = fields.TypedField('ldflags', typ.List[str]) #: Set compiler and compiler flags from the current programming environment #: if not specified otherwise. @@ -107,10 +107,10 @@ class BuildSystem(abc.ABC): flags_from_environ = fields.TypedField('flags_from_environ', bool) def __init__(self): - self.cc = None - self.cxx = None - self.ftn = None - self.nvcc = None + self.cc = '' + self.cxx = '' + self.ftn = '' + self.nvcc = '' self.cflags = [] self.cxxflags = [] self.cppflags = [] diff --git a/reframe/core/containers.py b/reframe/core/containers.py index afdda0486f..a0ec72086c 100644 --- a/reframe/core/containers.py +++ b/reframe/core/containers.py @@ -11,11 +11,7 @@ class ContainerPlatform(abc.ABC): - '''The abstract base class of any container platform. - - Concrete container platforms inherit from this class and must override the - :func:`emit_prepare_commands` and :func:`launch_command` abstract methods. - ''' + '''The abstract base class of any container platform.''' #: The container image to be used for running the test. #: @@ -74,6 +70,7 @@ def emit_prepare_commands(self): This method is relevant only to developers of new container platform backends. + :meta private: ''' @abc.abstractmethod @@ -85,6 +82,7 @@ def launch_command(self): This method is relevant only to developers of new container platforms. + :meta private: ''' def validate(self): diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py index 6e399fb7de..f4df1bfd89 100644 --- a/reframe/core/decorators.py +++ b/reframe/core/decorators.py @@ -129,13 +129,11 @@ def required_version(*versions): following formats: 1. ``VERSION``: Specifies a single version. - 2. ``{OP}VERSION``, where ``{OP}`` can be any of ``>``, ``>=``, ``<``, - ``<=``, ``==`` and ``!=``. For example, the version specification string - ``'>=2.15'`` will only allow the following test to be loaded only by - ReFrame 2.15 and higher. The ``==VERSION`` specification is the - equivalent of ``VERSION``. - + ``<=``, ``==`` and ``!=``. For example, the version specification + string ``'>=2.15'`` will allow the following test to be loaded only + by ReFrame 2.15 and higher. The ``==VERSION`` specification is the + equivalent of ``VERSION``. 3. ``V1..V2``: Specifies a range of versions. You can specify multiple versions with this decorator, such as @@ -190,9 +188,16 @@ def _fn(*args, **kwargs): def run_before(stage): - '''Run the decorated function before the specified pipeline stage. + '''Decorator for attaching a test method to a pipeline stage. + + The method will run just before the specified pipeline stage and it should + not accept any arguments except ``self``. + + This decorator can be stacked, in which case the function will be attached + to multiple pipeline stages. - The decorated function must be a method of a regression test. + The ``stage`` argument can be any of ``'setup'``, ``'compile'``, + ``'run'``, ``'sanity'``, ``'performance'`` or ``'cleanup'``. .. versionadded:: 2.20 @@ -201,9 +206,10 @@ def run_before(stage): def run_after(stage): - '''Run the decorated function after the specified pipeline stage. + '''Decorator for attaching a test method to a pipeline stage. - The decorated function must be a method of a regression test. + This is completely analogous to the + :py:attr:`reframe.core.decorators.run_before`. .. versionadded:: 2.20 @@ -220,7 +226,7 @@ def require_deps(func): :func:`reframe.core.pipeline.RegressionTest.getdep` function, such that conceptually the new function arguments will be the following: - .. code:: python + .. code-block:: python new_arg = functools.partial(getdep, orig_arg_name) diff --git a/reframe/core/environments.py b/reframe/core/environments.py index e8806fbc78..60282a2343 100644 --- a/reframe/core/environments.py +++ b/reframe/core/environments.py @@ -17,6 +17,9 @@ class Environment: It is simply a collection of modules to be loaded and environment variables to be set when this environment is loaded by the framework. + + .. warning:: + Users may not create :class:`Environment` objects directly. ''' def __init__(self, name, modules=None, variables=None): @@ -38,7 +41,7 @@ def name(self): def modules(self): '''The modules associated with this environment. - :type: :class:`list` of :class:`str` + :type: :class:`List[str]` ''' return util.SequenceView(self._modules) @@ -46,21 +49,10 @@ def modules(self): def variables(self): '''The environment variables associated with this environment. - :type: dictionary of :class:`str` keys/values. + :type: :class:`OrderedDict[str, str]` ''' return util.MappingView(self._variables) - def details(self): - '''Return a detailed description of this environment.''' - variables = '\n'.join(' '*8 + '- %s=%s' % (k, v) - for k, v in self.variables.items()) - lines = [ - self._name + ':', - ' modules: ' + ', '.join(self.modules), - ' variables:' + ('\n' if variables else '') + variables - ] - return '\n'.join(lines) - def __eq__(self, other): if not isinstance(other, type(self)): return NotImplemented @@ -80,6 +72,8 @@ def __repr__(self): class _EnvironmentSnapshot(Environment): + '''An environment snapshot.''' + def __init__(self, name='env_snapshot'): super().__init__(name, [], os.environ.items()) @@ -101,24 +95,21 @@ def __eq__(self, other): def snapshot(): - '''Create an environment snapshot''' + '''Create an environment snapshot + + :returns: An instance of :class:`_EnvironmentSnapshot`. + ''' return _EnvironmentSnapshot() class ProgEnvironment(Environment): '''A class representing a programming environment. - This type of environment adds also attributes for setting the compiler and - compilation flags. + This type of environment adds also properties for retrieving the compiler + and compilation flags. - If compilation flags are set to :class:`None` (the default, if not set - otherwise in ReFrame's `configuration - `__), they are not passed to the - ``make`` invocation. - - If you want to disable completely the propagation of the compilation flags - to the ``make`` invocation, even if they are set, you should set the - :attr:`propagate` attribute to :class:`False`. + .. warning:: + Users may not create :class:`ProgEnvironment` objects directly. ''' _cc = fields.TypedField('_cc', str) @@ -222,25 +213,3 @@ def ldflags(self): @property def nvcc(self): return self._nvcc - - def details(self): - def format_flags(flags): - if not flags: - return '' - else: - return ' '.join(flags) - - base_details = super().details() - extra_details = [ - ' CC: %s' % self.cc, - ' CXX: %s' % self.cxx, - ' FTN: %s' % self.ftn, - ' NVCC: %s' % self.nvcc, - ' CFLAGS: %s' % format_flags(self.cflags), - ' CXXFLAGS: %s' % format_flags(self.cxxflags), - ' FFLAGS: %s' % format_flags(self.fflags), - ' CPPFLAGS: %s' % format_flags(self.cppflags), - ' LDFLAGS: %s' % format_flags(self.ldflags) - ] - - return '\n'.join([base_details, '\n'.join(extra_details)]) diff --git a/reframe/core/launchers/__init__.py b/reframe/core/launchers/__init__.py index ecbae3d284..21c2767177 100644 --- a/reframe/core/launchers/__init__.py +++ b/reframe/core/launchers/__init__.py @@ -10,30 +10,25 @@ class JobLauncher(abc.ABC): - '''A job launcher. + '''Abstract base class for job launchers. A job launcher is the executable that actually launches a distributed program to multiple nodes, e.g., ``mpirun``, ``srun`` etc. - .. note:: + .. warning:: - Users cannot create job launchers directly. You may retrieve a - registered launcher backend through the - :func:`reframe.core.backends.getlauncher` function. + Users may not create job launchers directly. .. note:: .. versionchanged:: 2.8 Job launchers do not get a reference to a job during their initialization. - .. note:: - .. versionchanged:: 3.0 - The :func:`getlauncher` function has moved to a different module. ''' #: List of options to be passed to the job launcher invocation. #: - #: :type: :class:`list` of :class:`str` + #: :type: :class:`List[str]` #: :default: ``[]`` options = fields.TypedField('options', typ.List[str]) diff --git a/reframe/core/modules.py b/reframe/core/modules.py index c5bbd2bc70..4c64b9c825 100644 --- a/reframe/core/modules.py +++ b/reframe/core/modules.py @@ -78,11 +78,7 @@ def __str__(self): class ModulesSystem: - '''A modules system abstraction inside ReFrame. - - This class interfaces between the framework internals and the actual - modules systems implementation. - ''' + '''A modules system.''' module_map = fields.TypedField('module_map', types.Dict[str, types.List[str]]) @@ -114,6 +110,8 @@ def resolve_module(self, name): :returns: the list of real modules names pointed to by ``name``. :raises: :class:`reframe.core.exceptions.ConfigError` if the mapping contains a cycle. + + :meta private: ''' ret = OrderedSet() visited = set() @@ -157,7 +155,7 @@ def backend(self): def loaded_modules(self): '''Return a list of loaded modules. - This method returns a list of strings. + :rtype: List[str] ''' return [str(m) for m in self._backend.loaded_modules()] @@ -168,7 +166,7 @@ def conflicted_modules(self, name): list will be the concatenation of the conflict lists of all the real modules. - This method returns a list of strings. + :rtype: List[str] ''' ret = [] for m in self.resolve_module(name): @@ -186,7 +184,8 @@ def load_module(self, name, force=False): conflicting modules currently loaded. If module ``name`` refers to multiple real modules, all of the target modules will be loaded. - Returns the list of unloaded modules as strings. + :returns: the list of unloaded modules as strings. + :rtype: List[str] ''' ret = [] for m in self.resolve_module(name): @@ -243,6 +242,7 @@ def load_mapping(self, mapping): :arg mapping: a string specifying the module mapping. Example syntax: ``'m0: m1 m2'``. + :meta private: ''' key, *rest = mapping.split(':') if len(rest) != 1: @@ -259,7 +259,10 @@ def load_mapping(self, mapping): self.module_map[key] = list(OrderedDict.fromkeys(values)) def load_mapping_from_file(self, filename): - '''Update the internal module mappings from mappings read from file.''' + '''Update the internal module mappings from mappings read from file. + + :meta private: + ''' with open(filename) as fp: for lineno, line in enumerate(fp, start=1): line = line.strip().split('#')[0] @@ -273,12 +276,12 @@ def load_mapping_from_file(self, filename): @property def name(self): - '''Return the name of this module system.''' + '''The name of this module system.''' return self._backend.name() @property def version(self): - '''Return the version of this module system.''' + '''The version of this module system.''' return self._backend.version() def unload_all(self): @@ -299,13 +302,19 @@ def searchpath_remove(self, *dirs): return self._backend.searchpath_remove(*dirs) def emit_load_commands(self, name): - '''Return the appropriate shell command for loading module ``name``.''' + '''Return the appropriate shell command for loading module ``name``. + + :rtype: List[str] + ''' return [self._backend.emit_load_instr(Module(name)) for name in self.resolve_module(name)] def emit_unload_commands(self, name): '''Return the appropriate shell command for unloading module - ``name``.''' + ``name``. + + :rtype: List[str] + ''' return [self._backend.emit_unload_instr(Module(name)) for name in reversed(self.resolve_module(name))] diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 3182250703..ec1ce7f504 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -124,8 +124,7 @@ class RegressionTest(metaclass=RegressionTestMeta): .. note:: .. versionchanged:: 2.19 - - Base constructor takes no arguments. + Base constructor takes no arguments. ''' @@ -148,6 +147,7 @@ class RegressionTest(metaclass=RegressionTestMeta): #: #: .. versionchanged:: 2.17 #: Support for wildcards is dropped. + #: valid_prog_environs = fields.TypedField('valid_prog_environs', typ.List[str]) @@ -220,7 +220,7 @@ class RegressionTest(metaclass=RegressionTestMeta): #: #: This field may be set using either a string referring to a concrete #: build system class name - #: (see `build systems `__) or an instance of + #: (see `build systems <#build-systems>`__) or an instance of #: :class:`reframe.core.buildsystems.BuildSystem`. The former is the #: recommended way. #: @@ -233,9 +233,9 @@ class RegressionTest(metaclass=RegressionTestMeta): #: List of shell commands to be executed before compiling. #: - #: These commands are executed during the compilation phase and from - #: inside the stage directory. **Each entry in the list spawns a new - #: shell.** + #: These commands are emitted in the build script before the actual build + #: commands generated by the selected `build system + #: <#reframe.core.pipeline.RegressionTest.build_system>`__. #: #: :type: :class:`List[str]` #: :default: ``[]`` @@ -243,8 +243,9 @@ class RegressionTest(metaclass=RegressionTestMeta): #: List of shell commands to be executed after a successful compilation. #: - #: These commands are executed during the compilation phase and from inside - #: the stage directory. **Each entry in the list spawns a new shell.** + #: These commands are emitted in the script after the actual build + #: commands generated by the selected `build system + #: <#reframe.core.pipeline.RegressionTest.build_system>`__. #: #: :type: :class:`List[str]` #: :default: ``[]`` @@ -275,10 +276,9 @@ class RegressionTest(metaclass=RegressionTestMeta): #: self.container_platform.commands = ['cat /etc/os-release'] #: #: If this field is set, :attr:`executable` and :attr:`executable_opts` - #: attributes are ignored. The container platform's :attr:`commands` will - #: be used instead. For more information on the container platform support, - #: see the `tutorial `__ - #: and the `reference guide `__. + #: attributes are ignored. The container platform's :attr:`commands + #: ` will be used + #: instead. #: #: :type: :class:`str` or #: :class:`reframe.core.containers.ContainerPlatform`. @@ -366,15 +366,14 @@ class RegressionTest(metaclass=RegressionTestMeta): #: Number of tasks required by this test. #: - #: If the number of tasks is set to a number ``<=0``, ReFrame will try - #: to flexibly allocate the number of tasks, based on the command line - #: option ``--flex-alloc-nodes``. - #: A negative number is used to indicate the minimum number of tasks - #: required for the test. - #: In this case the minimum number of tasks is the absolute value of - #: the number, while - #: Setting ``num_tasks`` to ``0`` is equivalent to setting it to - #: ``-num_tasks_per_node``. + #: If the number of tasks is set to a number ``<=0``, ReFrame will try to + #: flexibly allocate the number of tasks, based on the command line option + #: |--flex-alloc-nodes|_. A negative number is used to indicate the minimum + #: number of tasks required for the test. In this case the minimum number + #: of tasks is the absolute value of the number, while Setting + #: :attr:`num_tasks` to ``0`` is equivalent to setting it to + #: :attr:`-num_tasks_per_node + #: `. #: #: :type: integral #: :default: ``1`` @@ -382,18 +381,13 @@ class RegressionTest(metaclass=RegressionTestMeta): #: .. note:: #: .. versionchanged:: 2.15 #: Added support for flexible allocation of the number of tasks - #: according to the ``--flex-alloc-tasks`` command line option - #: (see `Flexible node allocation - #: `__) #: if the number of tasks is set to ``0``. #: .. versionchanged:: 2.16 - #: Negative ``num_tasks`` is allowed for specifying the minimum + #: Negative :attr:`num_tasks` is allowed for specifying the minimum #: number of required tasks by the test. - #: .. versionchanged:: 2.21 - #: Flexible node allocation is now controlled by the - #: ``--flex-alloc-nodes`` command line option - #: (see `Flexible node allocation - #: `__) + #: + #: .. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` + #: .. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes num_tasks = fields.TypedField('num_tasks', int) #: Number of tasks per node required by this test. @@ -453,8 +447,7 @@ class RegressionTest(metaclass=RegressionTestMeta): #: :type: :class:`str` or :class:`datetime.timedelta` #: :default: :class:`None` #: - #: .. note:: - #: .. versionchanged:: 3.0 + #: .. versionadded:: 3.0 #: max_pending_time = fields.TimerField('max_pending_time', type(None)) @@ -499,14 +492,14 @@ class RegressionTest(metaclass=RegressionTestMeta): #: .. note:: #: .. versionchanged:: 3.0 #: The measurement unit is required. The user should explicitly - #: specify `None` if no unit is available. + #: specify :class:`None` if no unit is available. reference = fields.ScopedDictField( 'reference', typ.Tuple[object, object, object, object]) # FIXME: There is not way currently to express tuples of `float`s or # `None`s, so we just use the very generic `object` #: - #: Refer to the :doc:`ReFrame Tutorial ` for concrete usage + #: Refer to the :doc:`ReFrame Tutorials ` for concrete usage #: examples. #: #: If set to :class:`None`, a sanity error will be raised during sanity @@ -527,13 +520,12 @@ class RegressionTest(metaclass=RegressionTestMeta): #: :: #: #: self.sanity_patterns = sn.assert_found(r'.*', self.stdout) - #: sanity_patterns = fields.TypedField('sanity_patterns', _DeferredExpression, type(None)) #: Patterns for verifying the performance of this test. #: - #: Refer to the :doc:`ReFrame Tutorial ` for concrete usage + #: Refer to the :doc:`ReFrame Tutorials ` for concrete usage #: examples. #: #: If set to :class:`None`, no performance checking will be performed. @@ -571,30 +563,28 @@ class RegressionTest(metaclass=RegressionTestMeta): #: #: The value is internaly kept as a :class:`datetime.timedelta` object. #: For example '2h30m' is represented as - #: `datetime.timedelta(hours=2, minutes=30)` + #: ``datetime.timedelta(hours=2, minutes=30)`` #: #: :type: :class:`str` or :class:`datetime.timedelta` #: :default: ``'10m'`` #: #: .. note:: #: .. versionchanged:: 2.15 - #: - #: This attribute may be set to :class:`None`. + #: This attribute may be set to :class:`None`. #: #: .. warning:: #: .. versionchanged:: 3.0 - #: - #: The old syntax using a ``(h, m, s)`` tuple is deprecated. - #: + #: The old syntax using a ``(h, m, s)`` tuple is deprecated. time_limit = fields.TimerField('time_limit', type(None)) #: Extra resources for this test. #: - #: This field is for specifying custom resources needed by this test. - #: These resources are defined in the :doc:`configuration ` - #: of a system partition. - #: For example, assume that two additional resources, named ``gpu`` and - #: ``datawarp``, are defined in the configuration file as follows: + #: This field is for specifying custom resources needed by this test. These + #: resources are defined in the `configuration + #: `__ of a system + #: partition. For example, assume that two additional resources, named + #: ``gpu`` and ``datawarp``, are defined in the configuration file as + #: follows: #: #: :: #: @@ -650,9 +640,8 @@ class RegressionTest(metaclass=RegressionTestMeta): #: .. note:: #: .. versionadded:: 2.8 #: .. versionchanged:: 2.9 - #: - #: A new more powerful syntax was introduced - #: that allows also custom job script directive prefixes. + #: A new more powerful syntax was introduced + #: that allows also custom job script directive prefixes. extra_resources = fields.TypedField('extra_resources', typ.Dict[str, typ.Dict[str, object]]) @@ -793,7 +782,7 @@ def current_environ(self): This is set by the framework during the :func:`setup` phase. - :type: :class:`reframe.core.environments.Environment`. + :type: :class:`reframe.core.environments.ProgEnvironment`. ''' return self._current_environ @@ -803,7 +792,7 @@ def current_partition(self): This is set by the framework during the :func:`setup` phase. - :type: :class:`reframe.core.systems._SystemPartition`. + :type: :class:`reframe.core.systems.SystemPartition`. ''' return self._current_partition @@ -813,7 +802,7 @@ def current_system(self): This is set by the framework during the initialization phase. - :type: :class:`reframe.core.runtime.HostSystem`. + :type: :class:`reframe.core.systems.System`. ''' return rt.runtime().system @@ -922,12 +911,11 @@ def info(self): :returns: a string with an informational message about this test .. note :: + .. versionadded:: 2.10 + When overriding this method, you should pay extra attention on how you use the :class:`RegressionTest`'s attributes, because this method may be called at any point of the test's lifetime. - - .. versionadded:: 2.10 - ''' ret = self.name if self.current_partition: @@ -1026,12 +1014,12 @@ def setup(self, partition, environ, **job_opts): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' self._current_partition = partition self._current_environ = environ @@ -1063,12 +1051,12 @@ def compile(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' if not self._current_environ: raise PipelineError('no programming environment set') @@ -1161,12 +1149,12 @@ def compile_wait(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' self._build_job.wait() self.logger.debug('compilation finished') @@ -1185,12 +1173,11 @@ def run(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. ''' if not self.current_system or not self._current_partition: raise PipelineError('no system or system partition is set') @@ -1287,12 +1274,12 @@ def poll(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' if not self._job: return True @@ -1308,12 +1295,12 @@ def wait(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' self._job.wait() self.logger.debug('spawned job finished') @@ -1340,12 +1327,12 @@ def check_sanity(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' if self.sanity_patterns is None: raise SanityError('sanity_patterns not set') @@ -1364,12 +1351,12 @@ def check_performance(self): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' if self.perf_patterns is None: return @@ -1478,12 +1465,12 @@ def cleanup(self, remove_files=False): .. warning:: - You may not override this method directly unless you are in special - test. See `here - `__ for - more details. - .. versionchanged:: 3.0 + You may not override this method directly unless you are in + special test. See `here + `__ for + more details. + ''' aliased = os.path.samefile(self._stagedir, self._outputdir) if aliased: @@ -1521,7 +1508,7 @@ def depends_on(self, target, how=DEPEND_BY_ENV, subdeps=None): test cases of the target test ``T0``, but its ``E1`` case will depend only on the ``E1`` test case of ``T0``: - .. code:: python + .. code-block:: python self.depends_on('T0', how=rfm.DEPEND_EXACT, subdeps={'E0': ['E0', 'E1'], 'E1': ['E1']}) diff --git a/reframe/core/runtime.py b/reframe/core/runtime.py index 0846d95feb..1fcb6855a8 100644 --- a/reframe/core/runtime.py +++ b/reframe/core/runtime.py @@ -22,11 +22,6 @@ class RuntimeContext: '''The runtime context of the framework. - This class essentially groups the current host system and the associated - resources of the framework on the current system. - It also encapsulates other runtime parameters that are relevant to the - framework's execution. - There is a single instance of this class globally in the framework. .. note:: @@ -76,7 +71,7 @@ def site_config(self): def system(self): '''The current host system. - :type: :class:`reframe.core.runtime.HostSystem` + :type: :class:`reframe.core.systems.System` ''' return self._system @@ -117,7 +112,10 @@ def timestamp(self): @property def output_prefix(self): - '''The output prefix directory of ReFrame.''' + '''The output directory prefix. + + :type: :class:`str` + ''' if self.outputdir: ret = os.path.join(self.outputdir, self.timestamp) else: @@ -127,7 +125,10 @@ def output_prefix(self): @property def stage_prefix(self): - '''The stage prefix directory of ReFrame.''' + '''The stage directory prefix. + + :type: :class:`str` + ''' if self.stagedir: ret = os.path.join(self.stagedir, self.timestamp) else: @@ -145,13 +146,18 @@ def make_outputdir(self, *dirs, wipeout=True): @property def modules_system(self): - '''The modules system used by the current host system. + '''The environment modules system used in the current host. :type: :class:`reframe.core.modules.ModulesSystem`. ''' return self._system.modules_system def get_option(self, option): + '''Get a configuration option. + + :arg option: The option to be retrieved. + :returns: The value of the option. + ''' return self._site_config.get(option) @@ -167,9 +173,9 @@ def init_runtime(site_config): def runtime(): - '''Retrieve the framework's runtime context. + '''Get the runtime context of the framework. - :type: :class:`reframe.core.runtime.RuntimeContext` + :returns: A :class:`reframe.core.runtime.RuntimeContext` object. .. note:: .. versionadded:: 2.13 @@ -183,8 +189,14 @@ def runtime(): def loadenv(*environs): '''Load environments in the current Python context. - Returns a tuple containing a snapshot of the environment at entry to this - function and a list of shell commands required to load ``environs``. + :arg environs: A list of environments to load. + :type environs: List[Environment] + + :returns: A tuple containing snapshot of the current environment upon + entry to this function and a list of shell commands required to load + the environments. + :rtype: Tuple[_EnvironmentSnapshot, List[str]] + ''' modules_system = runtime().modules_system env_snapshot = snapshot() @@ -211,7 +223,13 @@ def emit_loadenv_commands(*environs): def is_env_loaded(environ): - ''':class:`True` if this environment is loaded, :class:`False` otherwise. + '''Check if environment is loaded. + + :arg environ: Environment to check for. + :type environ: Environment + + :returns: :class:`True` if this environment is loaded, :class:`False` + otherwise. ''' is_module_loaded = runtime().modules_system.is_module_loaded return (all(map(is_module_loaded, environ.modules)) and @@ -238,7 +256,10 @@ def __exit__(self, exc_type, exc_value, traceback): # The following utilities are useful only for the unit tests class temp_runtime: - '''Context manager to temporarily switch to another runtime.''' + '''Context manager to temporarily switch to another runtime. + + :meta private: + ''' def __init__(self, config_file, sysname=None, options=None): global _runtime_context @@ -265,7 +286,10 @@ def __exit__(self, exc_type, exc_value, traceback): def switch_runtime(config_file, sysname=None, options=None): '''Function decorator for temporarily changing the runtime for a - function.''' + function. + + :meta private: + ''' def _runtime_deco(fn): @functools.wraps(fn) def _fn(*args, **kwargs): @@ -280,7 +304,7 @@ def _fn(*args, **kwargs): class module_use: - '''Context manager for temporarily modifying the module path''' + '''Context manager for temporarily modifying the module path.''' def __init__(self, *paths): self._paths = paths diff --git a/reframe/core/schedulers/__init__.py b/reframe/core/schedulers/__init__.py index d7599d69da..6c992e1bfc 100644 --- a/reframe/core/schedulers/__init__.py +++ b/reframe/core/schedulers/__init__.py @@ -20,53 +20,85 @@ class JobScheduler(abc.ABC): + '''Abstract base class for job scheduler backends.''' + @abc.abstractmethod def completion_time(self, job): '''The completion time of this job expressed in seconds from the Epoch. + + :meta private: ''' - pass @abc.abstractmethod def emit_preamble(self, job): - pass + '''Return the job script preamble as a list of lines. + + :arg job: A job descriptor. + :returns: The job preamble as a list of lines. + :meta private: + ''' @abc.abstractmethod def allnodes(self): - '''Gets all the available nodes''' + '''Return a list of all the available nodes. + + :meta private: + ''' @abc.abstractmethod def filternodes(self, job, nodes): - '''Filter nodes according to the job options''' + '''Filter nodes according to job information. + + :arg job: A job descriptor. + :arg nodes: The initial set of nodes. + :returns: The filtered set of nodes. + :meta private: + ''' @abc.abstractmethod def submit(self, job): - pass + '''Submit a job. + + :arg job: A job descriptor. + :meta private: + ''' @abc.abstractmethod def wait(self, job): - pass + '''Wait for a job to finish. + + :arg job: A job descriptor. + :meta private: + ''' @abc.abstractmethod def cancel(self, job): - pass + '''Cancel a job. + + :arg job: A job descriptor. + :meta private: + ''' @abc.abstractmethod def finished(self, job): - pass + '''Poll a job. + + :arg job: A job descriptor. + :returns: :class:`True` if the job has finished, :class:`False` + otherwise. + + :meta private: + ''' class Job: '''A job descriptor. A job descriptor is created by the framework after the "setup" phase and - is associated with the test. It can be retrieved through the - :attr:`reframe.core.pipeline.RegressionTest.job` attribute and stores - information about the job submitted during the "run" phase. - - .. note:: + is associated with the test. - Users cannot create a job descriptor directly and associate it with a - test. + .. warning:: + Users may not create a job descriptor directly. ''' @@ -77,8 +109,9 @@ class Job: int, type(None)) num_tasks_per_socket = fields.TypedField('num_tasks_per_socket', int, type(None)) - num_cpus_per_tasks = fields.TypedField('num_cpus_per_task', - int, type(None)) + + num_cpus_per_task = fields.TypedField('num_cpus_per_task', + int, type(None)) use_smt = fields.TypedField('use_smt', bool, type(None)) time_limit = fields.TimerField('time_limit', type(None)) @@ -99,7 +132,7 @@ class Job: #: The following example shows how you can replace the current partition's #: launcher for this test with the "local" launcher: #: - #: .. code:: python + #: .. code-block:: python #: #: from reframe.core.backends import getlauncher #: @@ -113,7 +146,7 @@ class Job: #: The ID of the current job. #: - #: :type: :class:`int` or ``None``. + #: :type: :class:`int` or :class:`None`. #: #: .. versionadded:: 2.21 #: @@ -123,7 +156,7 @@ class Job: #: #: This may or may not be set depending on the scheduler backend. #: - #: :type: :class:`int` or ``None``. + #: :type: :class:`int` or :class:`None`. #: #: .. versionadded:: 2.21 #: @@ -133,7 +166,7 @@ class Job: #: #: The value of this field is scheduler-specific. #: - #: :type: :class:`str` or ``None``. + #: :type: :class:`str` or :class:`None`. #: #: .. versionadded:: 2.21 #: @@ -152,9 +185,8 @@ class Job: #: #: This attribute might be useful in a flexible regression test for #: determining the actual nodes that were assigned to the test. - #: For more information on flexible node allocation, please refer to the - #: corresponding `section `__ of - #: the tutorial. + #: For more information on flexible node allocation, see the + #: |--flex-alloc-nodes|_ command-line option #: #: This attribute is *not* supported by the ``pbs`` scheduler backend. #: @@ -221,7 +253,6 @@ def create(cls, scheduler, launcher, *args, **kwargs): ret.scheduler, ret.launcher = scheduler, launcher return ret - # Read-only properties @property def name(self): return self._name @@ -365,6 +396,11 @@ def finished(self): class Node(abc.ABC): + '''Abstract base class for representing system nodes. + + :meta private: + ''' + @abc.abstractmethod def is_available(self): '''Return ``True`` if this node is available, ``False`` otherwise.''' diff --git a/reframe/core/systems.py b/reframe/core/systems.py index 5c7930ee29..0b01de4a91 100644 --- a/reframe/core/systems.py +++ b/reframe/core/systems.py @@ -12,7 +12,13 @@ from reframe.core.environments import (Environment, ProgEnvironment) -class _SystemPartition: +class SystemPartition: + '''A representation of a system partition inside ReFrame. + + .. warning:: + Users may not create :class:`SystemPartition` objects directly. + ''' + def __init__(self, parent, name, scheduler, launcher, descr, access, container_environs, resources, local_env, environs, max_jobs): @@ -30,19 +36,36 @@ def __init__(self, parent, name, scheduler, launcher, @property def access(self): + '''The scheduler options for accessing this system partition. + + :type: :class:`List[str]` + ''' return utility.SequenceView(self._access) @property def descr(self): - '''A detailed description of this partition.''' + '''The description of this partition. + + :type: :class:`str` + ''' return self._descr @property def environs(self): + '''The programming environments associated with this system partition. + + :type: :class:`List[ProgEnvironment]` + ''' + return utility.SequenceView(self._environs) @property def container_environs(self): + '''Environments associated with the different container platforms. + + :type: :class:`Dict[str, Environment]` + ''' + return utility.MappingView(self._container_environs) @property @@ -52,28 +75,46 @@ def fullname(self): The fully-qualified name is of the form ``:``. - :type: `str` + :type: :class:`str` ''' return f'{self._parent_system}:{self._name}' @property def local_env(self): + '''The local environment associated with this partition. + + :type: :class:`Environment` + ''' return self._local_env @property def max_jobs(self): + '''The maximum number of concurrent jobs allowed on this partition. + + :type: integral + ''' return self._max_jobs @property def name(self): '''The name of this partition. - :type: `str` + :type: :class:`str` ''' return self._name @property def resources(self): + '''The resources template strings associated with this partition. + + This is a dictionary, where the key is the name of a resource and the + value is the scheduler options or directives associated with this + resource. + + :type: :class:`Dict[str, List[str]]` + + ''' + return utility.MappingView(self._resources) @property @@ -84,9 +125,9 @@ def scheduler(self): .. note:: .. versionchanged:: 2.8 + Prior versions returned a string representing the scheduler and + job launcher combination. - Prior versions returned a string representing the scheduler and job - launcher combination. ''' return self._scheduler @@ -101,8 +142,12 @@ def launcher(self): ''' return self._launcher - # Instantiate managed resource `name` with `value`. def get_resource(self, name, **values): + '''Instantiate managed resource ``name`` with ``value``. + + :meta private: + ''' + ret = [] for r in self._resources.get(name, []): try: @@ -113,6 +158,8 @@ def get_resource(self, name, **values): return ret def environment(self, name): + '''Return the partition environment named ``name``.''' + for e in self.environs: if e.name == name: return e @@ -132,6 +179,8 @@ def __eq__(self, other): self._local_env == other._local_env) def json(self): + '''Return a JSON object representing this system partition.''' + return { 'name': self._name, 'descr': self._descr, @@ -165,7 +214,11 @@ def __str__(self): class System: - '''A representation of a system inside ReFrame.''' + '''A representation of a system inside ReFrame. + + .. warning:: + Users may not create :class:`System` objects directly. + ''' def __init__(self, name, descr, hostnames, modules_system, preload_env, prefix, outputdir, @@ -224,7 +277,7 @@ def create(cls, site_config): ) for e in site_config.get(f'{partid}/environs') ] partitions.append( - _SystemPartition( + SystemPartition( parent=site_config.get('systems/0/name'), name=part_name, scheduler=part_sched, @@ -264,28 +317,42 @@ def create(cls, site_config): @property def name(self): - '''The name of this system.''' + '''The name of this system. + + :type: :class:`str` + ''' return self._name @property def descr(self): - '''The description of this system.''' + '''The description of this system. + + :type: :class:`str` + ''' return self._descr @property def hostnames(self): - '''The hostname patterns associated with this system.''' + '''The hostname patterns associated with this system. + + :type: :class:`List[str]` + ''' return self._hostnames @property def modules_system(self): - '''The modules system name associated with this system.''' + '''The modules system name associated with this system. + + :type: :class:`reframe.core.modules.ModulesSystem` + ''' return self._modules_system @property def preload_environ(self): '''The environment to load whenever ReFrame runs on this system. + :type: :class:`reframe.core.environments.Environment` + .. note:: .. versionadded:: 2.19 ''' @@ -293,35 +360,48 @@ def preload_environ(self): @property def prefix(self): - '''The ReFrame prefix associated with this system.''' + '''The ReFrame prefix associated with this system. + + :type: :class:`str` + ''' return self._prefix @property def stagedir(self): - '''The ReFrame stage directory prefix associated with this system.''' + '''The ReFrame stage directory prefix associated with this system. + + :type: :class:`str` + ''' return self._stagedir @property def outputdir(self): - '''The ReFrame output directory prefix associated with this system.''' + '''The ReFrame output directory prefix associated with this system. + + :type: :class:`str` + ''' return self._outputdir @property def resourcesdir(self): '''Global resources directory for this system. - You may use this directory for storing large resource files of your - regression tests. - See `here `__ on how to configure - this. + This directory may be used for storing large files related to + regression tests. The value of this directory is controlled by the + `resourcesdir `__ + configuration parameter. :type: :class:`str` + ''' return self._resourcesdir @property def partitions(self): - '''All the system partitions associated with this system.''' + '''The system partitions associated with this system. + + :type: :class:`List[SystemPartition]` + ''' return utility.SequenceView(self._partitions) def __eq__(self, other): @@ -333,6 +413,8 @@ def __eq__(self, other): self._partitions == other._partitions) def json(self): + '''Return a JSON object representing this system.''' + return { 'name': self._name, 'descr': self._descr, diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index c813e03e6c..359c77fb77 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -134,7 +134,7 @@ def main(): # Select options select_options.add_argument( - '-t', '--tag', action='append', dest='tags', default=[], + '-t', '--tag', action='append', dest='tags', metavar='TAG', default=[], help='Select checks matching TAG' ) select_options.add_argument( @@ -235,7 +235,7 @@ def main(): '-m', '--module', action='append', default=[], metavar='MOD', dest='user_modules', help='Load module MOD before running the regression suite', - envvar='RFM_USER_MODULES', configvar='general/user_modules' + envvar='RFM_USER_MODULES ,', configvar='general/user_modules' ) env_options.add_argument( '--module-mappings', action='store', metavar='FILE', @@ -247,7 +247,7 @@ def main(): '-u', '--unload-module', action='append', metavar='MOD', dest='unload_modules', default=[], help='Unload module MOD before running the regression suite', - envvar='RFM_UNLOAD_MODULES', configvar='general/unload_modules' + envvar='RFM_UNLOAD_MODULES ,', configvar='general/unload_modules' ) env_options.add_argument( '--purge-env', action='store_true', dest='purge_env', default=False, @@ -280,7 +280,7 @@ def main(): help='Print a report for performance tests run' ) misc_options.add_argument( - '--show-config-param', action='store', nargs='?', const='all', + '--show-config', action='store', nargs='?', const='all', metavar='PARAM', help=( 'Print how parameter PARAM is configured ' @@ -401,8 +401,8 @@ def main(): sys.exit(1) # Show configuration after everything is set up - if options.show_config_param: - config_param = options.show_config_param + if options.show_config: + config_param = options.show_config if config_param == 'all': printer.info(str(rt.site_config)) else: diff --git a/reframe/utility/sanity.py b/reframe/utility/sanity.py index 75d82444d8..d3a47f9af5 100644 --- a/reframe/utility/sanity.py +++ b/reframe/utility/sanity.py @@ -3,67 +3,6 @@ # # SPDX-License-Identifier: BSD-3-Clause -'''Sanity deferrable functions. - -This module provides functions to be used with the :attr:`sanity_patterns ` and -:attr`perf_patterns `. -The key characteristic of these functions is that they are not executed the -time they are called. Instead they are evaluated at a later point by the -framework (inside the :func:`check_sanity ` and :func:`check_performance ` methods). -Any sanity function may be evaluated either explicitly or implicitly. - -Explicit evaluation of sanity functions ---------------------------------------- - -Sanity functions may be evaluated at any time by calling the :func:`evaluate ` on their return value. - - -Implicit evaluation of sanity functions ---------------------------------------- - -Sanity functions may also be evaluated implicitly in the following situations: - -- When you try to get their truthy value by either explicitly or implicitly - calling :func:`bool ` on their return value. - This implies that when you include the result of a sanity function in an - :keyword:`if` statement or when you apply the :keyword:`and`, :keyword:`or` - or :keyword:`not` operators, this will trigger their immediate evaluation. -- When you try to iterate over their result. - This implies that including the result of a sanity function in a - :keyword:`for` statement will trigger its evaluation immediately. -- When you try to explicitly or implicitly get its string representation by - calling :func:`str ` on its result. - This implies that printing the return value of a sanity function will - automatically trigger its evaluation. - -This module provides three categories of sanity functions: - -1. Deferrable replacements of certain Python built-in functions. - These functions simply delegate their execution to the actual built-ins. -2. Assertion functions. - These functions are used to assert certain conditions and they either return - :class:`True` or raise :class:`reframe.core.exceptions.SanityError` with a - message describing the error. - Users may provide their own formatted messages through the ``msg`` - argument. - For example, in the following call to :func:`assert_eq` the ``{0}`` and - ``{1}`` placeholders will obtain the actual arguments passed to the - assertion function. - :: - - assert_eq(a, 1, msg="{0} is not equal to {1}") - - If in the user provided message more placeholders are used than the - arguments of the assert function (except the ``msg`` argument), no argument - substitution will be performed in the user message. -3. Utility functions. - The are functions that you will normally use when defining :attr:`sanity_patterns ` and :attr:`perf_patterns `. - They include, but are not limited to, functions to iterate over regex - matches in a file, extracting and converting values from regex matches, - computing statistical information on series of data etc. - -''' - import builtins import glob as pyglob import itertools @@ -89,25 +28,6 @@ def _format(s, *args, **kwargs): # Create an alias decorator sanity_function = deferrable -''':decorator: Sanity function decorator. - -Decorate any function to be used in sanity and/or performance patterns with -this decorator: -:: - - @sanity_function - def myfunc(*args): - do_sth() - -This decorator is an alias to the :func:`reframe.core.deferrable.deferrable` -decorator. -The following function definition is equivalent to the above: -:: - - @deferrable - def myfunc(*args): - do_sth() -''' # Deferrable versions of selected builtins @@ -441,7 +361,7 @@ def assert_found(patt, filename, msg=None, encoding='utf-8'): :arg patt: The regex pattern to search. Any standard Python `regular expression - `_ + `_ is accepted. :arg filename: The name of the file to examine. Any :class:`OSError` raised while processing the file will be @@ -581,12 +501,12 @@ def findall(patt, filename, encoding='utf-8'): :arg patt: The regex pattern to search. Any standard Python `regular expression - `_ + `_ is accepted. :arg filename: The name of the file to examine. :arg encoding: The name of the encoding used to decode the file. :returns: A list of raw `regex match objects - `_. + `_. :raises reframe.core.exceptions.SanityError: In case an :class:`OSError` is raised while processing ``filename``. ''' @@ -634,7 +554,7 @@ def extractall(patt, filename, tag=0, conv=None, encoding='utf-8'): :arg patt: The regex pattern to search. Any standard Python `regular expression - `_ + `_ is accepted. :arg filename: The name of the file to examine. :arg encoding: The name of the encoding used to decode the file. diff --git a/tutorial/advanced/advanced_example10.py b/tutorial/advanced/advanced_example10.py index a25a1454cb..d9168732d1 100644 --- a/tutorial/advanced/advanced_example10.py +++ b/tutorial/advanced/advanced_example10.py @@ -7,7 +7,6 @@ import reframe.utility.sanity as sn -@rfm.required_version('>=2.20-dev2') @rfm.simple_test class Example10Test(rfm.RunOnlyRegressionTest): def __init__(self): diff --git a/tutorial/advanced/advanced_example5.py b/tutorial/advanced/advanced_example5.py index 484ab6cb08..9dcc1585fb 100644 --- a/tutorial/advanced/advanced_example5.py +++ b/tutorial/advanced/advanced_example5.py @@ -10,7 +10,7 @@ @rfm.simple_test class TimeLimitTest(rfm.RunOnlyRegressionTest): def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the use' + self.descr = ('ReFrame tutorial demonstrating the use ' 'of a user-defined time limit') self.valid_systems = ['daint:gpu', 'daint:mc'] self.valid_prog_environs = ['*'] diff --git a/tutorial/advanced/advanced_example9.py b/tutorial/advanced/advanced_example9.py index b21e7464c7..182f218447 100644 --- a/tutorial/advanced/advanced_example9.py +++ b/tutorial/advanced/advanced_example9.py @@ -17,13 +17,8 @@ def __init__(self): self.num_tasks = 0 self.num_tasks_per_node = 1 self.sanity_patterns = sn.assert_eq( - self.num_tasks_assigned, + sn.getattr(self, 'num_tasks'), sn.count(sn.findall(r'nid\d+', self.stdout)) ) self.maintainers = ['you-can-type-your-email-here'] self.tags = {'tutorial'} - - @property - @sn.sanity_function - def num_tasks_assigned(self): - return self.job.num_tasks diff --git a/unittests/resources/checks/hellocheck_make.py b/unittests/resources/checks/hellocheck_make.py index 273d72abf8..757a37b38a 100644 --- a/unittests/resources/checks/hellocheck_make.py +++ b/unittests/resources/checks/hellocheck_make.py @@ -18,9 +18,8 @@ def __init__(self): self.build_system = 'Make' self.build_system.cflags = ['-O3'] self.build_system.cxxflags = ['-O3'] - self.build_system.makefile = 'Makefile.nofort' - self.executable = './hello_cpp' - self.keep_files = ['hello_cpp'] + self.executable = './hello_c' + self.keep_files = ['hello_c'] self.tags = {'foo', 'bar'} self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) self.maintainers = ['VK'] diff --git a/unittests/resources/checks/src/Makefile b/unittests/resources/checks/src/Makefile index eae49c24c0..f021faf9f4 100644 --- a/unittests/resources/checks/src/Makefile +++ b/unittests/resources/checks/src/Makefile @@ -1,17 +1,11 @@ .PHONY: all clean -PROGRAMS = hello_c hello_cpp hello_fort +PROGRAMS = hello_c all: $(PROGRAMS) hello_c: hello.c $(CC) $(CFLAGS) -o $@ $< -hello_cpp: hello.cpp - $(CXX) $(CXXFLAGS) -o $@ $< - -hello_fort: hello.f90 - $(FC) $(FFLAGS) -o $@ $< - clean: /bin/rm -f $(PROGRAMS) diff --git a/unittests/resources/checks/src/Makefile.nofort b/unittests/resources/checks/src/Makefile.nofort deleted file mode 100644 index 3e6dbe4bc3..0000000000 --- a/unittests/resources/checks/src/Makefile.nofort +++ /dev/null @@ -1,14 +0,0 @@ -.PHONY: all clean - -PROGRAMS = hello_c hello_cpp - -all: $(PROGRAMS) - -hello_c: hello.c - $(CC) $(CFLAGS) -o $@ $< - -hello_cpp: hello.cpp - $(CXX) $(CXXFLAGS) -o $@ $< - -clean: - /bin/rm -f $(PROGRAMS) diff --git a/unittests/resources/checks/src/hello.cpp b/unittests/resources/checks/src/hello.cpp deleted file mode 100644 index 57c83360c8..0000000000 --- a/unittests/resources/checks/src/hello.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include - -int main() -{ - std::cout << "Hello, World!\n"; - return 0; -} diff --git a/unittests/resources/checks/src/hello.f90 b/unittests/resources/checks/src/hello.f90 deleted file mode 100644 index 713cc2f878..0000000000 --- a/unittests/resources/checks/src/hello.f90 +++ /dev/null @@ -1,3 +0,0 @@ -program HelloWorld - write(*,*) "Hello, World!" -end program HelloWorld