From e722799f7a9d9f89b4d27c22cb5c1db73a8b2bd3 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 18 Jun 2020 00:16:57 +0200 Subject: [PATCH 1/7] WIP: Renew the tutorial --- docs/conf.py | 5 +- docs/regression_test_api.rst | 30 + docs/tutorial_advanced.rst | 510 -------- docs/tutorial_basic.rst | 733 ----------- docs/tutorial_basics.rst | 1089 +++++++++++++++++ docs/tutorial_misc_topics.rst | 395 ++++++ docs/tutorials.rst | 4 +- reframe/core/pipeline.py | 3 + tutorial/advanced/advanced_example1.py | 22 - tutorial/advanced/advanced_example2.py | 27 - tutorial/advanced/src/Makefile | 11 - tutorial/advanced/src/Makefile_example4 | 16 - tutorial/example4.py | 32 - ...ple_matrix_vector_multiplication_openacc.c | 76 -- .../basics/hello/hello1.py | 9 +- tutorials/basics/hello/hello2.py | 16 + tutorials/basics/hello/src/hello.c | 13 + tutorials/basics/hello/src/hello.cpp | 13 + tutorials/basics/hellomp/hellomp1.py | 19 + tutorials/basics/hellomp/hellomp2.py | 21 + tutorials/basics/hellomp/hellomp3.py | 22 + .../basics/hellomp/src/hello_threads.cpp | 49 + tutorials/basics/stream/stream1.py | 37 + tutorials/basics/stream/stream2.py | 45 + tutorials/basics/stream/stream3.py | 70 ++ tutorials/config/settings.py | 163 +++ .../example5.py => tutorials/misc/gpu/cuda.py | 12 +- .../misc/gpu/src/matvec.cu | 0 tutorials/misc/makefiles/maketest.py | 33 + tutorials/misc/makefiles/src/dotprod.cpp | 72 ++ .../misc/random/randint.py | 11 +- .../misc/random}/src/random_numbers.sh | 5 + tutorials/misc/runonly/echorand.py | 28 + 33 files changed, 2140 insertions(+), 1451 deletions(-) delete mode 100644 docs/tutorial_advanced.rst delete mode 100644 docs/tutorial_basic.rst create mode 100644 docs/tutorial_basics.rst create mode 100644 docs/tutorial_misc_topics.rst delete mode 100644 tutorial/advanced/advanced_example1.py delete mode 100644 tutorial/advanced/advanced_example2.py delete mode 100644 tutorial/advanced/src/Makefile delete mode 100644 tutorial/advanced/src/Makefile_example4 delete mode 100644 tutorial/example4.py delete mode 100644 tutorial/src/example_matrix_vector_multiplication_openacc.c rename tutorial/advanced/advanced_example3.py => tutorials/basics/hello/hello1.py (52%) create mode 100644 tutorials/basics/hello/hello2.py create mode 100644 tutorials/basics/hello/src/hello.c create mode 100644 tutorials/basics/hello/src/hello.cpp create mode 100644 tutorials/basics/hellomp/hellomp1.py create mode 100644 tutorials/basics/hellomp/hellomp2.py create mode 100644 tutorials/basics/hellomp/hellomp3.py create mode 100644 tutorials/basics/hellomp/src/hello_threads.cpp create mode 100644 tutorials/basics/stream/stream1.py create mode 100644 tutorials/basics/stream/stream2.py create mode 100644 tutorials/basics/stream/stream3.py create mode 100644 tutorials/config/settings.py rename tutorial/example5.py => tutorials/misc/gpu/cuda.py (64%) rename tutorial/src/example_matrix_vector_multiplication_cuda.cu => tutorials/misc/gpu/src/matvec.cu (100%) create mode 100644 tutorials/misc/makefiles/maketest.py create mode 100644 tutorials/misc/makefiles/src/dotprod.cpp rename tutorial/advanced/advanced_example6.py => tutorials/misc/random/randint.py (74%) rename {tutorial/advanced => tutorials/misc/random}/src/random_numbers.sh (50%) create mode 100644 tutorials/misc/runonly/echorand.py diff --git a/docs/conf.py b/docs/conf.py index ebcd3c143f..76c757e543 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -146,8 +146,8 @@ html_logo = "_static/img/reframe-logo-dark-bg.svg" -#autodoc_default_flags=['members', 'undoc-members', 'private-members', 'special-members', 'inherited-members', 'show-inheritance'] -#autodoc_default_flags=['members', 'undoc-members', 'inherited-members', 'show-inheritance'] +# autodoc_default_flags=['members', 'undoc-members', 'private-members', 'special-members', 'inherited-members', 'show-inheritance'] +# autodoc_default_flags=['members', 'undoc-members', 'inherited-members', 'show-inheritance'] autodoc_default_flags = ['members'] # End of READTHEDOCS @@ -179,7 +179,6 @@ # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ['_static'] - # Custom sidebar templates, must be a dictionary that maps document names # to template names. # diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index f79be64bd4..5ca6b11bc5 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -15,6 +15,7 @@ Regression Test Base Classes :show-inheritance: + Regression Test Class Decorators -------------------------------- @@ -193,3 +194,32 @@ The :py:mod:`reframe` module offers direct access to the basic test classes, con .. py:decorator:: reframe.simple_test See :func:`@reframe.core.decorators.simple_test `. + + + +.. _scheduler_options: + +Mapping of Test Attributes to Job Scheduler Backends +---------------------------------------------------- + +.. table:: + :align: left + + ============================ ============================= ========================================================================================== ================== + Test attribute Slurm option Torque option PBS option + ============================ ============================= ========================================================================================== ================== + :attr:`num_tasks` :obj:`--ntasks`:sup:`1` :obj:`-l nodes={num_tasks//num_tasks_per_node}:ppn={num_tasks_per_node*num_cpus_per_task}` :obj:`-l select={num_tasks//num_tasks_per_node}:mpiprocs={num_tasks_per_node}:ncpus={num_tasks_per_node*num_cpus_per_task}` + :attr:`num_tasks_per_node` :obj:`--ntasks-per-node` see :attr:`num_tasks` see :attr:`num_tasks` + :attr:`num_tasks_per_core` :obj:`--ntasks-per-core` n/a n/a + :attr:`num_tasks_per_socket` :obj:`--ntasks-per-socket` n/a n/a + :attr:`num_cpus_per_task` :obj:`--cpus-per-task` see :attr:`num_tasks` see :attr:`num_tasks` + :attr:`time_limit` :obj:`--time=hh:mm:ss` :obj:`-l walltime=hh:mm:ss` :obj:`-l walltime=hh:mm::ss` + :attr:`exclusive_access` :obj:`--exclusive` n/a n/a + :attr:`use_smt` :obj:`--hint=[no]multithread` n/a n/a + ============================ ============================= ========================================================================================== ================== + + +If any of the attributes is set to :class:`None` it will not be emitted at all in the job script. +In cases that the attribute is required, it will be set to ``1``. + +:sup:`1` The :obj:`--nodes` option may also be emitted if the :js:attr:`use_nodes_option ` scheduler configuration parameter is set. diff --git a/docs/tutorial_advanced.rst b/docs/tutorial_advanced.rst deleted file mode 100644 index ebbab1b7ed..0000000000 --- a/docs/tutorial_advanced.rst +++ /dev/null @@ -1,510 +0,0 @@ -.. |tutorialdir_adv| replace:: :obj:`tutorial/advanced/` -.. _tutorialdir_adv: https://github.com/eth-cscs/reframe/tree/master/tutorial/advanced -.. |tutorial_adv_example1| replace:: :obj:`tutorial/advanced/src/advanced_example1.c` -.. _tutorial_adv_example1: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/advanced_example1.c -.. |limits.sh| replace:: :obj:`scripts/limits.sh` -.. _limits.sh: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/scripts/limits.sh - - - -================================================= -Tutorial 2: Customizing Further a Regression Test -================================================= - -In this section, we are going to show some more elaborate use cases of ReFrame. -The corresponding scripts as well as the source code of the examples discussed here can be found in the directory |tutorialdir_adv|_. - - -Working with Makefiles ----------------------- - -We have already shown how you can compile a single source file associated with your regression test. -In this example, we show how ReFrame can leverage Makefiles to build executables. - -Compiling a regression test through a Makefile is straightforward with ReFrame. -If the :attr:`sourcepath ` attribute refers to a directory, then ReFrame will try to figure out the type of project and select the correct build system. -If it is not a CMake or Autotools based project, it will try to use ``make`` for building it. -More specifically, ReFrame first copies the :attr:`sourcesdir ` to the stage directory at the beginning of the compilation phase, then switches to the ``{stagedir}/{sourcepath}/`` and, finally, invokes ``make``. - -.. note:: - The :attr:`sourcepath ` attribute must be a relative path refering to a subdirectory of :attr:`sourcesdir `, i.e., relative paths starting with ``..`` will be rejected. - -By default, :attr:`sourcepath ` is the empty string and :attr:`sourcesdir ` is set to ``'src/'``, if such a directory exists. -As a result, by not specifying a :attr:`sourcepath ` at all, ReFrame will eventually compile the files found in the ``src/`` directory. -This is exactly what our first example here does. - -For completeness, here are the contents of ``Makefile`` provided: - -.. literalinclude:: ../tutorial/advanced/src/Makefile - :language: makefile - -The corresponding |tutorial_adv_example1|_ source file consists of a simple printing of a message, whose content depends on the preprocessor variable ``MESSAGE``: - -.. literalinclude:: ../tutorial/advanced/src/advanced_example1.c - :language: c - -The purpose of the regression test in this case is to set the preprocessor variable ``MESSAGE`` via ``CPPFLAGS`` and then check the standard output for the message ``SUCCESS``, which indicates that the preprocessor flag has been passed and processed correctly by the Makefile. - -The contents of this regression test are the following: - -.. literalinclude:: ../tutorial/advanced/advanced_example1.py - -The important bit here is how we set up the build system for this test: - -.. literalinclude:: ../tutorial/advanced/advanced_example1.py - :lines: 18-19 - :dedent: 4 - - -First, we set the build system to :attr:`Make ` and then set the preprocessor flags for the compilation. -ReFrame will invoke ``make`` as follows: - -.. code:: - - make -j 1 CC='cc' CXX='CC' FC='ftn' NVCC='nvcc' CPPFLAGS='-DMESSAGE' - -The compiler variables (``CC``, ``CXX`` etc.) are set based on the corresponding values specified in the `coniguration `__ of the current environment. -You may instruct the build system to ignore the default values from the environment by setting the following: - -.. code-block:: python - - self.build_system.flags_from_environ = False - -In this case, ``make`` will be invoked as follows: - -.. code:: - - make -j 1 CPPFLAGS='-DMESSAGE' - -Notice that the ``-j 1`` option is always generated. -You may change the maximum build concurrency as follows: - -.. code-block:: python - - self.build_system.max_concurrency = 4 - -By setting :attr:`max_concurrency ` to :class:`None`, no limit for concurrent parallel jobs will be placed. -This means that ``make -j`` will be used for building. - -Finally, you may also customize the name of the ``Makefile``. -You can achieve that by setting the corresponding variable of the :class:`Make ` build system: - -.. code-block:: python - - self.build_system.makefile = 'Makefile_custom' - - -More details on ReFrame's build systems, you may find `here `__. - - -Retrieving the source code from a Git repository -================================================ - -It might be the case that a regression test needs to clone its source code from a remote repository. -This can be achieved in two ways with ReFrame. -One way is to set the :attr:`sourcesdir` attribute to :class:`None` and explicitly clone or checkout a repository using the :attr:`prebuild_cmds `: - -.. code-block:: python - - self.sourcesdir = None - self.prebuild_cmds = ['git clone https://github.com/me/myrepo .'] - - -By setting :attr:`sourcesdir` to :class:`None`, you are telling ReFrame that you are going to provide the source files in the stage directory. -The working directory of the :attr:`prebuild_cmds` and :attr:`postbuild_cmds` commands will be the stage directory of the test. - - -An alternative way to retrieve specifically a Git repository is to assign its URL directly to the :attr:`sourcesdir` attribute: - -.. code-block:: python - - self.sourcesdir = 'https://github.com/me/myrepo' - -ReFrame will attempt to clone this repository inside the stage directory by executing ``git clone .`` and will then procede with the compilation as described above. - - -.. note:: - ReFrame recognizes only URLs in the :attr:`sourcesdir` attribute and requires passwordless access to the repository. - This means that the SCP-style repository specification will not be accepted. - You will have to specify it as URL using the ``ssh://`` protocol (see `Git documentation page `__). - - -Add a configuration step before compiling the code -================================================== - -It is often the case that a configuration step is needed before compiling a code with ``make``. -To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make" style of build systems. -It supports `CMake-based `__ projects through the :class:`CMake ` build system, as well as `Autotools-based `__ projects through the :class:`Autotools ` build system. - -For other build systems, you can achieve the same effect using the :class:`Make ` build system and the :attr:`prebuild_cmds ` for performing the configuration step. -The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``: - -.. code-block:: python - - self.prebuild_cmds = ['./custom_configure -with-mylib'] - self.build_system = 'Make' - self.build_system.cppflags = ['-DHAVE_FOO'] - self.build_system.flags_from_environ = False - -The generated build script then will have the following lines: - -.. code-block:: bash - - ./custom_configure -with-mylib - make -j 1 CPPFLAGS='-DHAVE_FOO' - - -Implementing a Run-Only Regression Test ---------------------------------------- - -There are cases when it is desirable to perform regression testing for an already built executable. -The following test uses the ``echo`` Bash shell command to print a random integer between specific lower and upper bounds. -Here is the full regression test: - -.. literalinclude:: ../tutorial/advanced/advanced_example2.py - -There is nothing special for this test compared to those presented `earlier `__ except that it derives from the :class:`RunOnlyRegressionTest ` and that it does not contain any resources (``self.sourcesdir = None``). -Note that run-only regression tests may also have resources, as for instance a precompiled executable or some input data. The copying of these resources to the stage directory is performed at the beginning of the run phase. -For standard regression tests, this happens at the beginning of the compilation phase, instead. -Furthermore, in this particular test the :attr:`executable ` consists only of standard Bash shell commands. -For this reason, we can set :attr:`sourcesdir ` to :class:`None` informing ReFrame that the test does not have any resources. - -.. note:: - .. versionchanged:: 3.0 - It is no more necessary to explicitly set :attr:`sourcesdir ` to :class:`None` for run-only tests without resources. - - -Implementing a Compile-Only Regression Test -------------------------------------------- - -ReFrame provides the option to write compile-only tests which consist only of a compilation phase without a specified executable. -This kind of tests must derive from the :class:`CompileOnlyRegressionTest ` class provided by the framework. -The following example reuses the code of our first example in this section and checks that no warnings are issued by the compiler: - -.. literalinclude:: ../tutorial/advanced/advanced_example3.py - -The important thing to note here is that the standard output and standard error of the tests, accessible through the :attr:`stdout ` and :attr:`stderr ` attributes, are now the corresponding those of the compilation command. -So sanity checking can be done in exactly the same way as with a normal test. - -Leveraging Environment Variables --------------------------------- - -We have already demonstrated in ":doc:`tutorial_basic`" that ReFrame allows you to load the required modules for regression tests and also set any needed environment variables. When setting environment variables for your test through the :attr:`variables ` attribute, you can assign them values of other, already defined, environment variables using the standard notation ``$OTHER_VARIABLE`` or ``${OTHER_VARIABLE}``. -The following regression test sets the ``CUDA_HOME`` environment variable to the value of the ``CUDATOOLKIT_HOME`` and then compiles and runs a simple program: - -.. literalinclude:: ../tutorial/advanced/advanced_example4.py - -Before discussing this test in more detail, let's first have a look in the source code and the Makefile of this example: - -.. literalinclude:: ../tutorial/advanced/src/advanced_example4.c - :language: c - -This program is pretty basic, but enough to demonstrate the use of environment variables from ReFrame. -It simply compares the value of the ``CUDA_HOME`` macro with the value of the environment variable ``CUDA_HOME`` at runtime, printing ``SUCCESS`` if they are not empty and match. -The Makefile for this example compiles this source by simply setting ``CUDA_HOME`` to the value of the ``CUDA_HOME`` environment variable: - -.. literalinclude:: ../tutorial/advanced/src/Makefile_example4 - :language: makefile - -Coming back now to the ReFrame regression test, the ``CUDATOOLKIT_HOME`` environment variable is defined by the ``cudatoolkit`` module. -If you try to run the test, you will see that it will succeed, meaning that the ``CUDA_HOME`` variable was set correctly both during the compilation and the runtime. -ReFrame will generate the following instructions in the shell scripts (build and run) associated with this test: - -.. code-block:: bash - - module load cudatoolkit - export CUDA_HOME=$CUDATOOLKIT_HOME - -Finally, as already mentioned `previously <#working-with-makefiles>`__, since the name of the makefile is not one of the standard ones, it must be set explicitly in the build system: - -.. literalinclude:: ../tutorial/advanced/advanced_example4.py - :lines: 21 - :dedent: 8 - -Setting a Time Limit for Regression Tests ------------------------------------------ - -ReFrame gives you the option to limit the execution time of regression tests. -The following example demonstrates how you can achieve this by limiting the execution time of a test that tries to sleep 100 seconds: - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - -The important bit here is the following line that sets the time limit for the test to one minute: - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 17 - :dedent: 8 - -The :attr:`time_limit ` attribute is a string in the form ``dhms`` and will impose a *runtime* limit to the associated job. -It will not kill the test if the job is pending for more time. -Time limits are implemented for all the scheduler backends. - -The sanity condition for this test verifies that associated job has been canceled due to the time limit (note that this message is SLURM-specific). - -.. literalinclude:: ../tutorial/advanced/advanced_example5.py - :lines: 20-21 - :dedent: 8 - - -.. note:: - .. versionadded:: 3.0 - The :attr:`max_pending_time ` attribute was added to force termination of a test if it is pending for too long. - - -Applying a sanity function iteratively --------------------------------------- - -It is often the case that a common sanity pattern has to be applied many times. -In this example we will demonstrate how the above situation can be easily tackled using the sanity functions offered by ReFrame. -Specifically, we would like to execute the following shell script and check that its output is correct: - -.. literalinclude:: ../tutorial/advanced/src/random_numbers.sh - :language: bash - -The above script simply prints 100 random integers between the limits given by the variables ``LOWER`` and ``UPPER``. -In the corresponding regression test we want to check that all the random numbers printed lie between 90 and 100 ensuring that the script executed correctly. -Hence, a common sanity check has to be applied to all the printed random numbers. -In ReFrame this can achieved by the use of :func:`map() ` sanity function accepting a function and an iterable as arguments. -Through :func:`map() ` the given function will be applied to all the members of the iterable object. -Note that since :func:`map() ` is a sanity function, its execution will be deferred. -The ReFrame test is the following: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - -First the random numbers are extracted through the :func:`extractall() ` function as follows: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 18-19 - :dedent: 8 - -The ``numbers`` variable is a deferred iterable, which upon evaluation will return all the extracted numbers. -In order to check that the extracted numbers lie within the specified limits, we make use of the :func:`map() ` sanity function, which will apply the :func:`assert_bounded() ` to all the elements of ``numbers``. - -There is still a small complication that needs to be addressed. -The :func:`all() ` function returns :class:`True` for empty iterables, which is not what we want. -So we must ensure that all the numbers are extracted as well. -To achieve this, we make use of :func:`count() ` to get the number of elements contained in ``numbers`` combined with :func:`assert_eq() ` to check that the number is indeed 100. -Finally, both of the above conditions have to be satisfied for the program execution to be considered successful, hence the use of the :func:`and_() ` function. -Note that the :keyword:`and` operator is not deferrable and will trigger the evaluation of any deferrable argument passed to it. -The full syntax for the :attr:`sanity_patterns` is the following: - -.. literalinclude:: ../tutorial/advanced/advanced_example6.py - :lines: 20-22 - :dedent: 8 - - -.. note:: - .. versionadded:: 2.13 - ReFrame offers also the :func:`allx() ` sanity function which, conversely to the builtin :func:`all()` function, will return :class:`False` if its iterable argument is empty. - - -Customizing the Generated Job Script ------------------------------------- - -It is often the case that you must run some commands before or after the parallel launch of your executable. -This can be easily achieved by using the :attr:`prerun_cmds ` and :attr:`postrun_cmds ` attributes of a ReFrame test. - -The following example is a slightly modified version of the previous one. -The lower and upper limits for the random numbers are now set inside a helper shell script in |limits.sh|_ and we want also to print the word ``FINISHED`` after our executable has finished. -In order to achieve this, we need to source the helper script just before launching the executable and ``echo`` the desired message just after it finishes. -Here is the test file: - -.. literalinclude:: ../tutorial/advanced/advanced_example7.py - -Notice the use of the :attr:`prerun_cmds` and :attr:`postrun_cmds` attributes. -These are lists of shell commands that are emitted verbatim in the job script. -The generated job script for this example is the following: - -.. code-block:: bash - - #!/bin/bash -l - #SBATCH --job-name="prerun_demo_check_daint_gpu_PrgEnv-gnu" - #SBATCH --time=0:10:0 - #SBATCH --ntasks=1 - #SBATCH --output=prerun_demo_check.out - #SBATCH --error=prerun_demo_check.err - #SBATCH --constraint=gpu - module load daint-gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - source scripts/limits.sh - srun ./random_numbers.sh - echo FINISHED - -ReFrame generates the job shell script using the following pattern: - -.. code-block:: bash - - #!/bin/bash -l - {job_scheduler_preamble} - {test_environment} - {prerun_cmds} - {parallel_launcher} {executable} {executable_opts} - {postrun_cmds} - -The ``job_scheduler_preamble`` contains the directives that control the job allocation. -The ``test_environment`` are the necessary commands for setting up the environment of the test. -This is the place where the modules and environment variables specified in :attr:`modules ` and :attr:`variables ` attributes are emitted. -Then the commands specified in :attr:`prerun_cmds ` follow, while those specified in the :attr:`postrun_cmds ` come after the launch of the parallel job. -The parallel launch itself consists of three parts: - -#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, -#. the regression test executable as specified in the :attr:`executable ` attribute and -#. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. - -A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. - - -Working with parameterized tests --------------------------------- - -.. versionadded:: 2.13 - -We have seen already in the `basic tutorial `__ how we could better organize the tests so as to avoid code duplication by using test class hierarchies. -An alternative technique, which could also be used in parallel with the class hierarchies, is to use `parameterized tests`. -The following is a test that takes a ``variant`` parameter, which controls which variant of the code will be used. -Depending on that value, the test is set up differently: - -.. literalinclude:: ../tutorial/advanced/advanced_example8.py - -If you have already gone through the :doc:`tutorial_basic`, this test can be easily understood. -The new bit here is the :func:`@parameterized_test ` decorator of the :class:`MatrixVectorTest` class. -This decorator takes an arbitrary number of arguments, which are either of a sequence type (i.e., list, tuple etc.) or of a mapping type (i.e., dictionary). -Each of the decorator's arguments corresponds to the constructor arguments of the decorated test that will be used to instantiate it. -In the example shown, the test will be instantiated twice, once passing ``variant`` as ``MPI`` and a second time with ``variant`` passed as ``OpenMP``. -The framework will try to generate unique names for the generated tests by stringifying the arguments passed to the test's constructor: - -.. code-block:: none - - [ReFrame Setup] - version: 3.0-dev6 (rev: 89d50861) - command: './bin/reframe -C tutorial/config/settings.py -c tutorial/advanced/advanced_example8.py -l' - launched by: karakasv@daint101 - working directory: '/path/to/reframe' - check search path: (R) '/path/to/reframe/tutorial/advanced/advanced_example8.py' - stage directory: '/path/to/reframe/stage' - output directory: '/path/to/reframe/output' - - [List of matched checks] - - MatrixVectorTest_MPI (found in /path/to/reframe/tutorial/advanced/advanced_example8.py) - - MatrixVectorTest_OpenMP (found in /path/to/reframe/tutorial/advanced/advanced_example8.py) - - Found 2 check(s). - - -There are a couple of different ways that we could have used the :func:`@parameterized_test ` decorator. -One is to use dictionaries for specifying the instantiations of our test class. -The dictionaries will be converted to keyword arguments and passed to the constructor of the test class: - -.. code-block:: python - - @rfm.parameterized_test({'variant': 'MPI'}, {'variant': 'OpenMP'}) - - -Another way, which is quite useful if you want to generate lots of different tests at the same time, is to use either `list comprehensions `__ or `generator expressions `__ for specifying the different test instantiations: - -.. code-block:: python - - @rfm.parameterized_test(*([variant] for variant in ['MPI', 'OpenMP'])) - - -.. tip:: - Combining parameterized tests and test class hierarchies can offer you a very flexible way for generating multiple related tests at once keeping at the same time the maintenance cost low. - We use this technique extensively in our tests. - - -Flexible Regression Tests -------------------------- - -.. versionadded:: 2.15 - -ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks ` attribute is set to a negative value or zero. -In ReFrame's terminology, such tests are called *flexible*. -Negative values indicate the minimum number of tasks that are acceptable for this test (a value of ``-4`` indicates that at least ``4`` tasks are required). -A zero value indicates the default minimum number of tasks which is equal to :attr:`num_tasks_per_node `. - -By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted with the |--flex-alloc-nodes|_ command-line option. -Flexible tests are very useful for diagnostics tests, e.g., tests for checking the health of a whole set nodes. -In this example, we demonstrate this feature through a simple test that runs ``hostname``. -The test will verify that all the nodes print the expected host name: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - -The first thing to notice in this test is that :attr:`num_tasks ` is set to zero. -This is a requirement for flexible tests: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 17 - :dedent: 8 - -The sanity function of this test simply counts the host names and verifies that they are as many as expected: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 19-22 - :dedent: 8 - -Notice, however, that the sanity check does not use :attr:`num_tasks` directly, but rather access the attribute through the :func:`sn.getattr() ` sanity function, which is a replacement for the :func:`getattr` builtin. -The reason for that is that at the time the sanity check expression is created, :attr:`num_tasks` is ``0`` and it will only be set to its actual value during the run phase. -Consequently, we need to defer the attribute retrieval, thus we use the :func:`sn.getattr() ` sanity function instead of accessing it directly - - -.. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` -.. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes - - - -Testing containerized applications ----------------------------------- - -.. versionadded:: 2.20 - - -ReFrame can be used also to test applications that run inside a container. -A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. -The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - -A container-based test in ReFrame requires that the :attr:`container_platform ` is set: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 17 - -This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. -In this case, the test will be using `Singularity `__ as a container platform. -If such a platform is not configured for the current system, the test will fail. -For a complete list of supported container platforms, the user is referred to the `configuration reference `__. - -As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 17-20 - -These two attributes are mandatory for container-based check. -The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. -It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. - -In the above example, ReFrame will run the container as follows: - -.. code:: shell - - singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' - -By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. -The user commands then are then run from that directory one after the other. -Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. - -Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 21 - -Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: - -.. code-block:: python - - self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), - ('/path/to/host/dir2', '/path/to/container/mount_point2')] - - -For a complete list of the available attributes of a specific container platform, the reader is referred to `ReFrame Programming APIs `__ guide. diff --git a/docs/tutorial_basic.rst b/docs/tutorial_basic.rst deleted file mode 100644 index 0d8bd85c9b..0000000000 --- a/docs/tutorial_basic.rst +++ /dev/null @@ -1,733 +0,0 @@ -.. |tutorialdir| replace:: :obj:`tutorial/` -.. _tutorialdir: https://github.com/eth-cscs/reframe/tree/master/tutorial -.. |tutorial_settings| replace:: :obj:`tutorial/config/settings.py` -.. _tutorial_settings: https://github.com/eth-cscs/reframe/blob/master/tutorial/config/settings.py -.. |tutorial_matvec| replace:: :obj:`tutorial/src/example_matrix_vector_multiplication.c` -.. _tutorial_matvec: https://github.com/eth-cscs/reframe/blob/master/tutorial/src/example_matrix_vector_multiplication.c -.. |tutorial_matvec_hybrid| replace:: :obj:`tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c` -.. _tutorial_matvec_hybrid: https://github.com/eth-cscs/reframe/blob/master/tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c - - -====================== -Tutorial 1: The Basics -====================== - -This tutorial will guide you through writing your first regression tests with ReFrame. -We will start with the most common and simple case of a regression test that compiles a code, runs it and checks its output. -We will then expand this example gradually by adding functionality and more advanced sanity and performance checks. -By the end of the tutorial, you should be able to start writing your first regression tests with ReFrame. - -All the tutorial examples can be found under the |tutorialdir|_ directory in ReFrame's source code. -Regardless how you have `installed `__ ReFrame, you can get the tutorial examples by cloning the `project `__ on Github. - -This tutorial is tailored to the Piz Daint supercomputer, but you can use the tutorial tests on your system with slight adaptations. -The configuration file for the tutorial can be found in |tutorial_settings|_ and we list it also here for completeness: - -.. literalinclude:: ../tutorial/config/settings.py - :lines: 10- - - -The First Regression Test -------------------------- - -The following is a simple regression test that compiles and runs a serial C program, which computes a matrix-vector product (|tutorial_matvec|_), and verifies its sane execution. -As a sanity check, it simply looks for a specific output in the output of the program. -Here is the full code for this test: - -.. literalinclude:: ../tutorial/example1.py - -A regression test written in ReFrame is essentially a Python class that must eventually derive from :class:`RegressionTest `. -To make a test visible to the framework, you must decorate your final test class with one of the following decorators: - -* :func:`@simple_test `: for registering a single parameterless instantiation of your test. -* :func:`@parameterized_test `: for registering multiple instantiations of your test. - -Let's see in more detail how the ``Example1Test`` is defined: - -.. literalinclude:: ../tutorial/example1.py - :lines: 9-11 - -The ``__init__()`` method is the constructor of your test. -It is usually the only method you need to implement for your tests, especially if you don't want to customize any of the regression test pipeline stages. -When your test is instantiated, the framework assigns a default name to it. -This name is essentially a concatenation of the fully qualified name of the class and string representations of the constructor arguments, with any non-alphanumeric characters converted to underscores. -In this example, the auto-generated test name is simply ``Example1Test``. -You may change the name of the test later in the constructor by setting the :attr:`name ` attribute. - -.. note:: - .. versionchanged:: 2.19 - Calling ``super().__init__()`` inside the constructor of a test is no more needed. - - -.. warning:: - .. versionadded:: 2.12 - ReFrame requires that the names of all the tests it loads are unique. - In case of name clashes, it will refuse to load the conflicting test. - -The next line sets a more detailed description of the test: - -.. literalinclude:: ../tutorial/example1.py - :lines: 12 - :dedent: 8 - -This is optional and it defaults to the auto-generated name of the test, if not specified. - -.. note:: - If you explicitly set only the name of the test, the description will not be automatically updated and will still keep its default value. - -The next two lines specify the systems and the programming environments that this test is valid for: - -.. literalinclude:: ../tutorial/example1.py - :lines: 13-14 - :dedent: 8 - -Both of these variables accept a list of system names or environment names, respectively. -The ``*`` symbol is a wildcard meaning any system or any programming environment. -The system and environment names listed in these variables must correspond to names of systems and environments defined in the ReFrame's :doc:`configuration file `. - -.. note:: - - If a name specified in these lists does not appear in the settings file, it will be simply ignored. - -When specifying system names you can always specify a partition name as well by appending ``:`` to the system's name. -For example, given the configuration for our tutorial, ``daint:gpu`` would refer specifically to the ``gpu`` virtual partition of the system ``daint``. -If only a system name (without a partition) is specified in the :attr:`self.valid_systems ` variable, e.g., ``daint``, it means that this test is valid for any partition of this system. - -The next line specifies the source file that needs to be compiled: - -.. literalinclude:: ../tutorial/example1.py - :lines: 15 - :dedent: 8 - -ReFrame expects any source files, or generally resources, of the test to be inside an ``src/`` directory, which is at the same level as the regression test file. -If you inspect the directory structure of the |tutorialdir|_ folder, you will notice that: - -.. code-block:: none - - tutorial/ - example1.py - src/ - example_matrix_vector_multiplication.c - -Notice also that you need not specify the programming language of the file you are asking ReFrame to compile or the compiler to use. -ReFrame will automatically pick the correct compiler based on the extension of the source file. -The exact compiler that is going to be used depends on the programming environment that the test is running with. -For example, given our configuration, if it is run with ``PrgEnv-cray``, the Cray C compiler will be used, if it is run with ``PrgEnv-gnu``, the GCC compiler will be used etc. -A user can associate compilers with programming environments in the ReFrame's :doc:`configuration file `. - -The next line in our first regression test specifies a list of options to be used for running the generated executable (the matrix dimension and the number of iterations in this particular example): - -.. literalinclude:: ../tutorial/example1.py - :lines: 16 - :dedent: 8 - -Notice that you do not need to specify the executable name. -Since ReFrame compiled it and generated it, it knows the name. -We will see in ":doc:`tutorial_advanced`" how you can specify the name of the executable, in cases that ReFrame cannot guess its name. - -The next lines specify what should be checked for assessing the sanity of the result of the test: - -.. literalinclude:: ../tutorial/example1.py - :lines: 17-18 - :dedent: 8 - -This expression simply asks ReFrame to look for ``time for single matrix vector multiplication`` in the standard output of the test. -The :attr:`sanity_patterns ` attribute can only be assigned the result of a special type of functions, called *sanity functions*. -`Sanity functions `__ are special in the sense that they are evaluated lazily. -You can generally treat them as normal Python functions inside a :attr:`sanity_patterns ` expression. -ReFrame provides already a wide range of useful sanity functions ranging from wrappers to the standard built-in functions of Python to functions related to parsing the output of a regression test. -For a complete listing of the available functions, you may have a look at the :doc:`sanity_functions_reference`. - -In our example, the :func:`assert_found ` function accepts a regular expression pattern to be searched in a file and either returns :class:`True` on success or raises a :class:`SanityError ` in case of failure with a descriptive message. -This function accepts any valid `Python Regular Expression `__. -As a file argument, :func:`assert_found ` accepts any filename, which will be resolved against the stage directory of the test. -You can also use the :attr:`stdout ` and :attr:`stderr ` attributes to reference the standard output and standard error, respectively. - -.. tip:: You don't need to care about handling exceptions, and error handling in general, inside your test. - The framework will automatically abort the execution of the test, report the error and continue with the next test case. - -The last two lines of the regression test are optional, but serve a good role in a production environment: - -.. literalinclude:: ../tutorial/example1.py - :lines: 19-20 - :dedent: 8 - -In the :attr:`maintainers ` attribute you may store a list of persons responsible for the maintenance of this test. -In case of failure, this list will be printed in the failure summary. - -The :attr:`tags ` attribute is a set of tags that you can assign to this test. -This is useful for categorizing the tests and helps in quickly selecting the tests of interest. - -.. note:: The values assigned to the attributes of a :class:`RegressionTest ` are validated and if they don't have the correct type, an error will be issued by ReFrame. - For a list of all the attributes and their types, please refer to the :doc:`regression_test_api` guide. - - ------------------------------ -Running the Tutorial Examples ------------------------------ - -ReFrame offers a rich `command-line interface `__ that allows you to control several aspects of its executions. -Here we will only show you how to run a specific tutorial test: - -.. code-block:: bash - - ./bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r - -If everything is configured correctly for your system, you should get an output similar to the following: - -.. code-block:: none - - [ReFrame Setup] - version: 3.0-dev7 (rev: 85ca676f) - command: './bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r' - launched by: user@daint104 - working directory: '/path/to/reframe' - settings file: 'tutorial/config/settings.py' - check search path: (R) '/path/to/reframe/tutorial/example1.py' - stage directory: '/path/to/reframe/stage' - output directory: '/path/to/reframe/output' - - [==========] Running 1 check(s) - [==========] Started on Wed Jun 3 08:50:46 2020 - - [----------] started processing Example1Test (Simple matrix-vector multiplication example) - [ RUN ] Example1Test on daint:login using PrgEnv-cray - [ RUN ] Example1Test on daint:login using PrgEnv-gnu - [ RUN ] Example1Test on daint:login using PrgEnv-intel - [ RUN ] Example1Test on daint:login using PrgEnv-pgi - [ RUN ] Example1Test on daint:gpu using PrgEnv-cray - [ RUN ] Example1Test on daint:gpu using PrgEnv-gnu - [ RUN ] Example1Test on daint:gpu using PrgEnv-intel - [ RUN ] Example1Test on daint:gpu using PrgEnv-pgi - [ RUN ] Example1Test on daint:mc using PrgEnv-cray - [ RUN ] Example1Test on daint:mc using PrgEnv-gnu - [ RUN ] Example1Test on daint:mc using PrgEnv-intel - [ RUN ] Example1Test on daint:mc using PrgEnv-pgi - [----------] finished processing Example1Test (Simple matrix-vector multiplication example) - - [----------] waiting for spawned checks to finish - [ OK ] ( 1/12) Example1Test on daint:login using PrgEnv-intel [compile: 1.940s run: 20.747s total: 23.778s] - [ OK ] ( 2/12) Example1Test on daint:login using PrgEnv-cray [compile: 0.347s run: 25.096s total: 26.591s] - [ OK ] ( 3/12) Example1Test on daint:mc using PrgEnv-intel [compile: 2.019s run: 6.286s total: 8.357s] - [ OK ] ( 4/12) Example1Test on daint:mc using PrgEnv-cray [compile: 0.506s run: 11.056s total: 11.744s] - [ OK ] ( 5/12) Example1Test on daint:gpu using PrgEnv-cray [compile: 0.435s run: 19.499s total: 20.483s] - [ OK ] ( 6/12) Example1Test on daint:login using PrgEnv-gnu [compile: 1.648s run: 25.631s total: 27.964s] - [ OK ] ( 7/12) Example1Test on daint:mc using PrgEnv-gnu [compile: 1.825s run: 10.434s total: 12.301s] - [ OK ] ( 8/12) Example1Test on daint:gpu using PrgEnv-intel [compile: 2.018s run: 16.316s total: 18.529s] - [ OK ] ( 9/12) Example1Test on daint:login using PrgEnv-pgi [compile: 1.643s run: 22.118s total: 24.090s] - [ OK ] (10/12) Example1Test on daint:gpu using PrgEnv-gnu [compile: 1.729s run: 20.028s total: 21.901s] - [ OK ] (11/12) Example1Test on daint:gpu using PrgEnv-pgi [compile: 1.753s run: 15.128s total: 16.923s] - [ OK ] (12/12) Example1Test on daint:mc using PrgEnv-pgi [compile: 1.732s run: 35.556s total: 37.330s] - [----------] all spawned checks have finished - - [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) - [==========] Finished on Wed Jun 3 08:51:46 2020 - - -Notice how our regression test is run on every partition of the configured system and for every programming environment. - -Now that you have got a first understanding of how a regression test is written in ReFrame, let's try to expand our example. - --------------------------------------- -Inspecting the ReFrame Generated Files --------------------------------------- - -ReFrame generates several files during the execution of a test. -When developing or debugging a regression test it is important to be able to locate them and inspect them. - -As soon as the `setup` stage of the test is executed, a stage directory specific to this test is generated. -All the required resources for the test are copied to this directory, and this will be the working directory for the compilation, running, sanity and performance checking phases. -If the test is successful, this stage directory is removed, unless the :option:`--keep-stage-files` option is passed in the command line. -Before removing this directory, ReFrame copies the following files to a dedicated output directory for this test: - -- The generated build script and its standard output and standard error. - This allows you to inspect exactly how your test was compiled. -- The generated run script and its standard output and standard error. - This allows you to inspect exactly how your test was run and verify that the sanity checking was correct. -- Any other user-specified files. - -If a regression test fails, its stage directory will not be removed. -This allows you to reproduce exactly what ReFrame was trying to perform and will help you debug the problem with your test. - -Let's rerun our first example and instruct ReFrame to keep the stage directory of the test, so that we can inspect it. - -.. code:: - - ./bin/reframe -C tutorial/config/settings.py -c tutorial/example1.py -r --keep-stage-files - -ReFrame creates a stage directory for each test case using the following pattern: - -.. code:: - - $STAGEDIR_PREFIX//// - -Let's pick the test case for the ``gpu`` partition and the ``PrgEnv-gnu`` programming environment from our first test to inspect. -The default ``STAGEDIR_PREFIX`` is ``./stage``: - -.. code:: - - cd stage/daint/gpu/PrgEnv-gnu/Example1Test/ - -If you do a listing in this directory, you will see all the files contained in the ``tutorial/src`` directory, as well as the following files: - -.. code:: - - rfm_Example1Test_build.err rfm_Example1Test_job.err - rfm_Example1Test_build.out rfm_Example1Test_job.out - rfm_Example1Test_build.sh rfm_Example1Test_job.sh - - -The ``rfm_Example1Test_build.sh`` is the generated build script and the ``.out`` and ``.err`` are the compilation's standard output and standard error. -Here is the generated build script for our first test: - -.. code:: shell - - #!/bin/bash - - _onerror() - { - exitcode=$? - echo "-reframe: command \`$BASH_COMMAND' failed (exit code: $exitcode)" - exit $exitcode - } - - trap _onerror ERR - - module load daint-gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - cc example_matrix_vector_multiplication.c -o ./Example1Test - - -Similarly, the ``rfm_Example1Test_job.sh`` is the generated job script and the ``.out`` and ``.err`` files are the corresponding standard output and standard error. -The generated job script for the test case we are currently inspecting is the following: - -.. code:: shell - - #!/bin/bash -l - #SBATCH --job-name="rfm_Example1Test_job" - #SBATCH --time=0:10:0 - #SBATCH --ntasks=1 - #SBATCH --output=rfm_Example1Test_job.out - #SBATCH --error=rfm_Example1Test_job.err - #SBATCH --constraint=gpu - module load daint-gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - srun ./Example1Test 1024 100 - - -It is interesting to check here the generated job script for the ``login`` partition of the example system, which does not use a workload manager: - -.. code:: - - cat stage/daint/login/PrgEnv-gnu/Example1Test/rfm_Example1Test_job.sh - -.. code:: shell - - #!/bin/bash -l - module unload PrgEnv-cray - module load PrgEnv-gnu - ./Example1Test 1024 100 - - -This is one of the advantages in using ReFrame: -You do not have to care about the system-level details of the target system that your test is running. -Based on its configuration, ReFrame will generate the appropriate commands to run your test. - - -Customizing the Compilation Phase ---------------------------------- - -In this example, we write a regression test to compile and run the OpenMP version of the matrix-vector product program, that we have shown before. -The full code of this test follows: - -.. literalinclude:: ../tutorial/example2.py - :lines: 6-38 - -This example introduces two new concepts: - -1. We need to set the ``OMP_NUM_THREADS`` environment variable, in order to specify the number of threads to use with our program. -2. We need to specify different flags for the different compilers provided by the programming environments we are testing. - Notice also that we now restrict the validity of our test only to the programming environments that we know how to handle (see the :attr:`valid_prog_environs `). - -To define environment variables to be set during the execution of a test, you should use the :attr:`variables ` attribute of the regression test. -This is a dictionary, whose keys are the names of the environment variables and whose values are the values of the environment variables. -Notice that both the keys and the values must be strings. - -From version 2.14, ReFrame manages compilation of tests through the concept of build systems. -Any customization of the build process should go through a build system. -For straightforward cases, as in our first example, where no customization is needed, ReFrame automatically picks the correct build system to build the code. -In this example, however, we want to set the flags for compiling the OpenMP code. -Assuming our test supported only GCC, we could simply add the following lines in the ``__init__()`` method of our test: - -.. code:: python - - self.build_system = 'SingleSource' - self.build_system.cflags = ['-fopenmp'] - -The :class:`SingleSource ` build system that we use here supports the compilation of a single file only. -Each build system type defines a set of variables that the user can set. -Based on the selected build system, ReFrame will generate a build script that will be used for building the code. -The generated build script can be found in the stage or the output directory of the test, along with the output of the compilation. -This way, you may reproduce exactly what ReFrame does in case of any errors. -More on the build systems feature can be found `here `__. - -Getting back to our test, simply setting the ``cflags`` to ``-fopenmp`` globally in the test will make it fail for programming environments other than ``PrgEnv-gnu``, since the OpenMP flags vary for the different compilers. -Ideally, we need to set the ``cflags`` differently for each programming environment. -To achieve this we need to define a method that will set the compilation flags based on the current programming environment (i.e., the environment that test currently runs with) and schedule it to run before the ``compile`` stage of the test pipeline as follows: - -.. literalinclude:: ../tutorial/example2.py - :lines: 28-38 - :dedent: 4 - -In this function we retrieve the current environment from the :attr:`current_environ ` attribute, so we can then differentiate the build system's flags based on its name. -Note that we cannot retrieve the current programming environment inside the test's constructor, since as described in in the ":doc:`pipeline`" page, it is during the setup phase that a regression test is prepared for a new system partition and a new programming environment. -The second important thing in this function is the ``@run_before('compile')`` decorator. -This decorator will attach this function to the ``compile`` stage of the pipeline and will execute it before entering this stage. -Similarly to the :func:`@run_before ` decorator, there is also the :func:`@run_after `, which will run the decorated function after the specified pipeline stage. -The decorated function may have any name, but it should be a method of the test taking no arguments (i.e., its sole argument should the ``self``). - -You may attach multiple functions to the same stage as in the following example: - -.. code:: python - - @rfm.run_before('compile') - def setflags(self): - env = self.current_environ.name - if env == 'PrgEnv-cray': - self.build_system.cflags = ['-homp'] - elif env == 'PrgEnv-gnu': - self.build_system.cflags = ['-fopenmp'] - elif env == 'PrgEnv-intel': - self.build_system.cflags = ['-openmp'] - elif env == 'PrgEnv-pgi': - self.build_system.cflags = ['-mp'] - - @rfm.run_before('compile') - def set_more_flags(self): - self.build_system.flags += ['-g'] - -In this case, the decorated functions will be executed before the compilation stage in the order that they are defined in the regression test. - -There is also the possibility to attach a single function to multiple stages by stacking the :func:`@run_before ` or :func:`@run_after ` decorators. -In the following example ``var`` will be set to ``2`` after the setup phase is executed: - -.. code:: python - - def __init__(self): - ... - self.var = 0 - - - @rfm.run_before('setup') - @rfm.run_after('setup') - def inc(self): - self.var += 1 - -Another important feature of the hooks syntax, is that hooks are inherited by derived tests, unless you override the function and re-hook it explicitly. -In the following example, the :func:`setflags()` will be executed before the compilation phase of the :class:`DerivedTest`: - -.. code:: python - - class BaseTest(rfm.RegressionTest): - def __init__(self): - ... - self.build_system = 'Make' - - @rfm.run_before('compile') - def setflags(self): - if self.current_environ.name == 'X': - self.build_system.cppflags = ['-Ifoo'] - - - @rfm.simple_test - class DerivedTest(BaseTest): - def __init__(self): - super().__init__() - ... - - -If you override a hooked function in a derived class, the base class' hook will not be executed, unless you explicitly call it with ``super()``. -In the following example, we completely disable the :func:`setflags()` hook of the base class: - - -.. code:: python - - @rfm.simple_test - class DerivedTest(BaseTest): - @rfm.run_before('compile') - def setflags(self): - pass - - -Notice that in order to redefine a hook, you need not only redefine the method in the derived class, but you should hook it at the same pipeline phase. -Otherwise, the base class hook will be executed. - - -.. warning:: - .. versionchanged:: 3.0 - Configuring your test per programming environment and per system partition by overriding the :func:`setup ` method is deprecated. - Please refer to the ":doc:`migration_2_to_3`" guide for more details. - - -.. warning:: - .. versionchanged:: 2.17 - Support for setting the compiler flags in the programming environment has been dropped completely. - - - ------------------------------------------------- -An alternative implementation using dictionaries ------------------------------------------------- - -Here we present an alternative implementation of the same test using a dictionary to hold the compilation flags for the different programming environments. -The advantage of this implementation is that you move the different compilation flags in the initialization phase, where also the rest of the test's specification is, thus making it more concise. - -The :func:`setflags` method is now very simple: -it gets the correct compilation flags from the :attr:`prgenv_flags` dictionary and applies them to the build system. - -.. literalinclude:: ../tutorial/example2.py - :lines: 6-9,41-67 - -.. tip:: - A regression test is like any other Python class, so you can freely define your own attributes. - If you accidentally try to write on a reserved :class:`RegressionTest ` attribute that is not writeable, ReFrame will prevent this and it will throw an error. - -Running on Multiple Nodes -------------------------- - -So far, all our tests run on a single node. -Depending on the actual system that ReFrame is running, the test may run locally or be submitted to the system's job scheduler. -In this example, we write a regression test for the MPI+OpenMP version of the matrix-vector product. -The source code of this program is in |tutorial_matvec_hybrid|_. -The regression test file follows: - -.. literalinclude:: ../tutorial/example3.py - -This test is pretty much similar to the `test example <#an-alternative-implementation-using-dictionaries>`__ for the OpenMP code we have shown before, except that it adds some information about the configuration of the distributed tasks. -It also restricts the valid systems only to those that support distributed execution. -Let's take the changes step-by-step: - -First we need to specify for which partitions this test is meaningful by setting the :attr:`valid_systems ` attribute: - -.. literalinclude:: ../tutorial/example3.py - :lines: 14 - :dedent: 8 - -We only specify the partitions that are configured with a job scheduler. -If we try to run the generated executable on the login nodes, it will fail. -So we remove this partition from the list of the supported systems. - -The most important addition to this check are the variables controlling the distributed execution: - -.. literalinclude:: ../tutorial/example3.py - :lines: 28-30 - :dedent: 8 - -By setting these variables, we specify that this test should run with 8 MPI tasks in total, using two tasks per node. -Each task may use four logical CPUs. -Based on these variables ReFrame will generate the appropriate scheduler flags to meet that requirement. -For example, for Slurm these variables will result in the following flags: -``--ntasks=8``, ``--ntasks-per-node=2`` and ``--cpus-per-task=4``. -ReFrame provides several more variables for configuring the job submission. -As shown in the following Table, they follow closely the corresponding Slurm options. -For schedulers that do not provide the same functionality, some of the variables may be ignored. - -.. table:: - :align: center - - ================================= ========================== - :class:`RegressionTest` attribute Corresponding SLURM option - ================================= ========================== - ``time_limit = '10m30s`` ``--time=00:10:30`` - ``use_multithreading = True`` ``--hint=multithread`` - ``use_multithreading = False`` ``--hint=nomultithread`` - ``exclusive_access = True`` ``--exclusive`` - ``num_tasks=72`` ``--ntasks=72`` - ``num_tasks_per_node=36`` ``--ntasks-per-node=36`` - ``num_cpus_per_task=4`` ``--cpus-per-task=4`` - ``num_tasks_per_core=2`` ``--ntasks-per-core=2`` - ``num_tasks_per_socket=36`` ``--ntasks-per-socket=36`` - ================================= ========================== - - -Testing a GPU Code ------------------- - -In this example, we will create two regression tests for two different GPU versions of our matrix-vector code: -OpenACC and CUDA. -Let's start with the OpenACC regression test: - -.. literalinclude:: ../tutorial/example4.py - -The things to notice in this test are the restricted list of system partitions and programming environments that this test supports and the use of the :attr:`modules ` variable: - -.. literalinclude:: ../tutorial/example4.py - :lines: 19 - :dedent: 8 - -The :attr:`modules ` variable takes a list of environment modules that should be loaded during the setup phase of the test. -In this particular test, we need to load the ``craype-accel-nvidia60`` module, which enables the generation of a GPU binary from an OpenACC code. - -It's advisable for GPU-enabled tests to define also the :attr:`num_gpus_per_node ` attribute, since this information may be needed by some system partitions in order to get the right nodes: - -.. literalinclude:: ../tutorial/example4.py - :lines: 20 - :dedent: 8 - -The regression test for the CUDA code is slightly simpler: - -.. literalinclude:: ../tutorial/example5.py - -ReFrame will recognize the ``.cu`` extension of the source file and it will try to invoke ``nvcc`` for compiling the code. -In this case, there is no need to differentiate across the programming environments, since the compiler will be eventually the same. -``nvcc`` in our example is provided by the ``cudatoolkit`` module, which we list it in the :attr:`modules ` variable. - -More Advanced Sanity Checking ------------------------------ - -So far we have done a very simple sanity checking. -We are only looking if a specific line is present in the output of the test program. -In this example, we expand the regression test of the serial code, so as to check also if the printed norm of the result vector is correct. - -.. literalinclude:: ../tutorial/example6.py - -The only difference with our first example is actually the more complex expression to assess the sanity of the test. -Let's go over it line-by-line. -The first thing we do is to extract the norm printed in the standard output. - -.. literalinclude:: ../tutorial/example6.py - :lines: 23-25 - :dedent: 8 - -The :func:`extractsingle ` sanity function extracts some information from a single occurrence (by default the first) of a pattern in a filename. -In our case, this function will extract the ``norm`` `capturing group `__ from the match of the regular expression ``r'The L2 norm of the resulting vector is:\s+(?P\S+)'`` in standard output, it will convert it to float and it will return it. -Unnamed capturing groups in regular expressions are also supported, which you can reference by their group number. -For example, we could have written the same statement as follows: - -.. code-block:: python - - found_norm = sn.extractsingle( - r'The L2 norm of the resulting vector is:\s+(\S+)', - self.stdout, 1, float) - -Notice that we replaced the ``'norm'`` argument with ``1``, which is the capturing group number. - -.. note:: In regular expressions, capturing group ``0`` corresponds always to the whole match. - In sanity functions dealing with regular expressions, this will yield the whole line that matched. - -A useful counterpart of :func:`extractsingle ` is the :func:`extractall ` function, which instead of a single occurrence, returns a list of all the occurrences found. -For a more detailed description of this and other sanity functions, please refer to the `sanity function reference `__. - -The next four lines is the actual sanity check: - -.. literalinclude:: ../tutorial/example6.py - :lines: 26-30 - :dedent: 8 - -This expression combines two conditions that need to be true, in order for the sanity check to succeed: - -1. Find in standard output the same line we were looking for already in the first example. -2. Verify that the printed norm does not deviate significantly from the expected value. - -The :func:`all() ` function is responsible for combining the results of the individual subexpressions. -It is essentially the Python built-in :func:`all() ` function, exposed as a sanity function, and requires that all the elements of the iterable it takes as an argument evaluate to :class:`True`. -As mentioned before, all the :func:`assert_*` functions either return :class:`True` on success or raise :class:`SanityError `. -So, if everything goes smoothly, :func:`sn.all() ` will evaluate to :class:`True` and sanity checking will succeed. - -The expression for the second condition is more interesting. -Here, we want to assert that the absolute value of the difference between the expected and the found norm are below a certain value. -The important thing to mention here is that you can combine the results of sanity functions in arbitrary expressions, use them as arguments to other functions, return them from functions, assign them to variables etc. -Remember that sanity functions are not evaluated at the time you call them. -They will be evaluated later by the framework during the sanity checking phase. -If you include the result of a sanity function in an expression, the evaluation of the resulting expression will also be deferred. -For a detailed description of the mechanism behind the sanity functions, please have a look at the ":doc:`deferrables`" page. - -Writing a Performance Test --------------------------- - -An important aspect of regression testing is checking for performance regressions. -ReFrame offers a flexible way of extracting and manipulating performance data from the program output, as well as a comprehensive way of setting performance thresholds per system and system partitions. - -In this example, we extend the CUDA test presented `previously <#testing-a-gpu-code>`__, so as to check also the performance of the matrix-vector multiplication. - -.. literalinclude:: ../tutorial/example7.py - -The are two new variables set in this test that basically enable the performance testing: - -- :attr:`perf_patterns ` - This variable defines which are the performance patterns we are looking for and how to extract the performance values. -- :attr:`reference ` - This variable is a collection of reference values for different systems. - -Let's have a closer look at each of them: - -.. literalinclude:: ../tutorial/example7.py - :lines: 24-27 - :dedent: 8 - -The :attr:`perf_patterns ` attribute is a dictionary, whose keys are *performance variables* (i.e., arbitrary names assigned to the performance values we are looking for), and its values are *sanity expressions* that specify how to obtain these performance values from the output. -A sanity expression is a Python expression that uses the result of one or more *sanity functions*. -In our example, we name the performance value we are looking for simply as ``perf`` and we extract its value by converting to float the regex capturing group named ``Gflops`` from the line that was matched in the standard output. - -Each of the performance variables defined in :attr:`perf_patterns ` must be resolved in the :attr:`reference ` dictionary of reference values. -When the framework obtains a performance value from the output of the test it searches for a reference value in the :attr:`reference ` dictionary, and then it checks whether the user supplied tolerance is respected. -Let's go over the :attr:`reference ` dictionary of our example and explain its syntax in more detail: - -.. literalinclude:: ../tutorial/example7.py - :lines: 28-32 - :dedent: 8 - -This is a special type of dictionary that we call *scoped dictionary*, because it defines scopes for its keys. -In order to resolve a reference value for a performance variable, ReFrame creates the following key ``::`` and looks it up inside the :attr:`reference ` dictionary. -If our example, since this test is only allowed to run on the ``daint:gpu`` partition of our system, ReFrame will look for the ``daint:gpu:perf`` reference key. -The ``perf`` subkey will then be searched in the following scopes in this order: -``daint:gpu``, ``daint``, ``*``. -The first occurrence will be used as the reference value of the ``perf`` performance variable. -In our example, the ``perf`` key will be resolved in the ``daint:gpu`` scope giving us the reference value. - -Reference values in ReFrame are specified as a four-tuple comprising the reference value, the lower and upper thresholds and the measurement unit. -If no unit is relevant, then you have to insert :class:`None` explicitly. -Thresholds are specified as decimal fractions of the reference value. For nonnegative reference values, the lower threshold must lie in the [-1,0], whereas the upper threshold may be any positive real number or zero. -In our example, the reference value for this test on ``daint:gpu`` is 50 Gflop/s ±10%. Setting a threshold value to :class:`None` disables the threshold. -If you specify a measurement unit as well, you will be able to log it in the performance logs of the test; this is handy when you are inspecting or plotting the performance values. - -ReFrame will always add a default ``*`` entry in the ``reference`` dictionary, if it does not exist, with the reference value of ``(0, None, None, )``, where ``unit`` is derived from the unit of each respective performance variable. -This is useful when using ReFrame for benchmarking purposes and you would like to run a test on an unknown system. - -.. note:: - .. versionadded:: 2.16 - Reference tuples may now optionally contain units. - - .. versionadded:: 2.19 - A default ``*`` entry is now always added to the reference dictionary. - - .. versionchanged:: 3.0 - Reference tuples now require the measurement unit. - - -Combining It All Together -------------------------- - -As we have mentioned before and as you have already experienced with the examples in this tutorial, regression tests in ReFrame are written in pure Python. -As a result, you can leverage the language features and capabilities to organize better your tests and decrease the maintenance cost. -In this example, we are going to reimplement all the tests of the tutorial with much less code and in a single file. -Here is the final example code that combines all the tests discussed before: - -.. literalinclude:: ../tutorial/example8.py - -This test abstracts away the common functionality found in almost all of our tutorial tests (executable options, sanity checking, etc.) to a base class, from which all the concrete regression tests derive. -Each test then redefines only the parts that are specific to it. -Notice also that only the actual tests, i.e., the derived classes, are made visible to the framework through the :func:`@simple_test ` decorator. -Decorating the base class has no meaning, because it does not correspond to an actual test. - -The total line count of this refactored example is less than half of that of the individual tutorial tests. -Another interesting thing to note here is the base class accepting additional additional parameters to its constructor, so that the concrete subclasses can initialize it based on their needs. - -Summary -------- - -This concludes the first ReFrame tutorial. -We have covered all basic aspects of writing regression tests in ReFrame and you should now be able to start experimenting by writing your first useful tests. -The `next tutorial `__ covers further topics in customizing a regression test to your needs. diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst new file mode 100644 index 0000000000..2906b76d7b --- /dev/null +++ b/docs/tutorial_basics.rst @@ -0,0 +1,1089 @@ +========================================== + Tutorial 1: Getting Started with ReFrame +========================================== + +.. versionadded:: 3.1 + +.. |tutorialdir| replace:: :obj:`tutorials/` +.. |tutorialdir_basics| replace:: :obj:`tutorials/basics` +.. _tutorialdir_basics: https://github.com/eth-cscs/reframe/tree/master/tutorials/basics + +This tutorial will give you a first overview of ReFrame and will acquaint with its basic concepts. +We will start with a simple "Hello, World!" test running with the default configuration and we will expand the example along the way. +We will also explore performance tests and we will port our tests to an HPC cluster. +The examples of this tutorial can be found in |tutorialdir_basics|_. + + +Getting Ready +------------- + +All you need to start off with this tutorial is to have `installed `__ ReFrame. +If you haven't done so yet, all you need is Python 3.6 and above and to follow the steps below: + + +.. code:: bash + + git clone https://github.com/eth-cscs/reframe.git + cd reframe + ./bootstrap.sh + ./bin/reframe -V + +We're now good to go! + + +The "Hello, World!" test +------------------------ + +As simple as it may sound, a series of "naive" "Hello, World!" tests can reveal lots of regressions in the programming environment of HPC clusters, but the bare minimum of those also serves perfectly the purpose of starting this tutorial. +Here is its C version: + +.. literalinclude:: ../tutorials/basics/hello/src/hello.c + :lines: 6- + + +And here is the ReFrame version of it: + +.. literalinclude:: ../tutorials/basics/hello/hello1.py + :lines: 6- + + +Regression tests in ReFrame are specially decorated classes that ultimately derive from :class:`RegressionTest `. +The :func:`@simple_test ` decorator registers a test class with ReFrame and makes it available to the framework. +The test parameters are essentially attributes of the test class and are usually defined in the test class constructor (:func:`__init__` function). +Each test must always set the :attr:`valid_systems ` and :attr:`valid_prog_environs ` attributes. +These define the systems and/or system partitions that this test is allowed to run on, as well as the programming environments that it is valid for. +A programming environment is essentially a compiler toolchain. +We will see later on in the tutorial how a programming environment can be defined. +The generic configuration of ReFrame assumes a single programming environment named ``builtin`` which comprises a C compiler that can be invoked with ``cc``. +In this particular test we set both these attributes to ``['*']``, essentially allowing this test to run everywhere. + +Each regression test must always define the :attr:`sanity_patterns ` attribute. +This is a `lazily evaluated `__ expression that asserts the sanity of the test. +In this particular case, we ask ReFrame to check for the desired phrase in the test's standard output. +Note that ReFrame does not determine the success of a test by its exit code. +The assessment of success is responsibility of the test itself. + +Finally, a test must either define an executable to execute or a source file (or source code) to be compiled. +In this example, it is enough to define the source file of our hello program. +ReFrame knows the executable that was produced and will use that to run the test. + + +Before running the test let's inspect the directory structure surrounding it: + +.. code-block:: none + + tutorials/basics/hello + ├── hello1.py + └── src + └── hello.c + +Our test is ``hello1.py`` and its resources, i.e., the ``hello.c`` source file, are located inside the ``src/`` subdirectory. +If not specified otherwise, the :attr:`sourcepath ` attribute is always resolved relative to ``src/``. +There is full flexibility in organizing the tests. +Multiple tests may be defined in a single file or they may be split in multiple files. +Similarly, several tests may share the same resources directory or they can simply have their own. + +Now it's time to run our first test: + +.. code:: bash + + ./bin/reframe -c tutorials/basics/hello/hello1.py -r + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 986c3505) + command: './bin/reframe -c tutorials/basics/hello/hello1.py -r' + launched by: user@tresa.local + working directory: '/Users/user/reframe' + settings file: '' + check search path: '/Users/user/reframe/tutorials/basics/hello/hello1.py' + stage directory: '/Users/user/reframe/stage' + output directory: '/Users/user/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Sat Jun 20 09:44:52 2020 + + [----------] started processing HelloTest (HelloTest) + [ RUN ] HelloTest on generic:default using builtin + [----------] finished processing HelloTest (HelloTest) + + [----------] waiting for spawned checks to finish + [ OK ] (1/1) HelloTest on generic:default using builtin [compile: 0.735s run: 0.505s total: 1.272s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 1 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Sat Jun 20 09:44:53 2020 + + +Perfect! We have verified that we have a functioning C compiler in our system. + +When ReFrame runs a test, it copies all its resources to a stage directory and performs all test-related operations (compilation, run, sanity checking etc.) from that directory. +On successful outcome of the test, the stage directory is removed by default, but interesting files are copied to an output directory for archiving and later inspection. +The prefixes of these directories are printed in the first section of the output. +Let's inspect what files ReFrame produced for this test: + +.. code-block:: bash + + ls output/generic/default/builtin/HelloTest/ + +.. code-block:: none + + rfm_HelloTest_build.err rfm_HelloTest_build.sh rfm_HelloTest_job.out + rfm_HelloTest_build.out rfm_HelloTest_job.err rfm_HelloTest_job.sh + +ReFrame stores in the output directory of the test the build and "job" scripts it generated for building and running the code along with their standard output and error. +All these files are prefixed with ``rfm_``. + + +More of "Hello, World!" +----------------------- + +We want to extend our test and run a C++ "Hello, World!" as well. +We could simply copy paste the ``hello1.py`` and change the source file extension to refer to the C++ source code. +But this duplication is something that we generally want to avoid. +ReFrame allows you to avoid this in several ways but the most compact is to define the new test as follows: + + +.. literalinclude:: ../tutorials/basics/hello/hello2.py + :lines: 6- + + +This exactly the same test as the ``hello1.py`` except that it is decorated with the :func:`@parameterized_test ` decorator instead of the :func:`@simple_test `. +Also the constructor of the test now takes an argument. +The :func:`@parameterized_test <>` decorator instructs ReFrame to instantiate a test class with different parameters. +In this case the test will be instantiated for both C and C++ and then we use the ``lang`` parameter directly as the extension of the source file. +Let's run now the test: + + +.. code-block:: console + + ./bin/reframe -c tutorials/basics/hello/hello2.py -r + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 986c3505) + command: './bin/reframe -c tutorials/basics/hello/hello2.py -r' + launched by: user@tresa.local + working directory: '/Users/user/reframe' + settings file: '' + check search path: '/Users/user/reframe/tutorials/basics/hello/hello2.py' + stage directory: '/Users/user/reframe/stage' + output directory: '/Users/user/reframe/output' + + [==========] Running 2 check(s) + [==========] Started on Sat Jun 20 23:28:32 2020 + + [----------] started processing HelloMultiLangTest_c (HelloMultiLangTest_c) + [ RUN ] HelloMultiLangTest_c on generic:default using builtin + [----------] finished processing HelloMultiLangTest_c (HelloMultiLangTest_c) + + [----------] started processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + [ RUN ] HelloMultiLangTest_cpp on generic:default using builtin + [ HOLD ] HelloMultiLangTest_cpp on generic:default using builtin + [----------] finished processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + + [----------] waiting for spawned checks to finish + [ OK ] (1/2) HelloMultiLangTest_c on generic:default using builtin [compile: 1.068s run: 0.330s total: 1.431s] + [ FAIL ] (2/2) HelloMultiLangTest_cpp on generic:default using builtin [compile: 0.002s run: n/a total: 0.340s] + [----------] all spawned checks have finished + + [ FAILED ] Ran 2 test case(s) from 2 check(s) (1 failure(s)) + [==========] Finished on Sat Jun 20 23:28:33 2020 + + ============================================================================== + SUMMARY OF FAILURES + ------------------------------------------------------------------------------ + FAILURE INFO for HelloMultiLangTest_cpp + * Test Description: HelloMultiLangTest_cpp + * System partition: generic:default + * Environment: builtin + * Stage directory: /Users/user/reframe/stage/generic/default/builtin/HelloMultiLangTest_cpp + * Node list: + * Job type: local (id=None) + * Maintainers: [] + * Failing phase: compile + * Rerun with '-n HelloMultiLangTest_cpp -p builtin --system generic:default' + * Reason: build system error: I do not know how to compile a C++ program + ------------------------------------------------------------------------------ + + +Oops! The C++ test has failed. +ReFrame complains that it does not know how to compile a C++ program. +Remember our discussion above that the default configuration of ReFrame defines a minimal programming environment named ``builtin`` which only knows of a ``cc`` compiler. +We will fix that in a moment, but before doing that it's worth looking into the failure information provided for the test. +For each test failing, ReFrame will print a short summary with information about the system partition and the programming environment that the test failed for, its job or process id (if any), the nodes it was running on, its stage directory, the phase that failed etc. + +When a test fails its stage directory is kept intact, so that users can inspect the failure and try to reproduce it manually. +In this case, the stage directory contains only the "Hello, World" source files, since ReFrame could not produce a build script for the C++ test, as it doesn't know to compile a C++ program for the moment. + +.. code-block:: console + + ls stage/generic/default/builtin/HelloMultiLangTest_cpp + + +.. code-block:: none + + hello.c hello.cpp + + +Let's go on and fix this failure by defining a new system and programming environments for the machine we are running on. +We start off by copying the generic configuration file that ReFrame uses. +Note that you should *not* edit this configuration file in place. + +.. code-block:: console + + cp reframe/core/settings.py tutorials/config/mysettings.py + + +Here is how the new configuration file looks like with the needed additions highlighted: + +.. literalinclude:: ../tutorials/config/settings.py + :lines: 10-25,59-79,112- + :emphasize-lines: 3-16,32-43 + +Here we define a system named ``catalina`` that has one partition named ``default``. +This partition makes no use of any workload manager, but instead launches any jobs locally as OS processes. +Two programming environments are relevant for that partition, namely ``gnu`` and ``clang``, which are defined in the section :js:attr:`environments` of the configuration file. +The ``gnu`` programming environment provides GCC 9, whereas the ``clang`` one provides the Clang compiler from the system. +Notice, how you can define the actual commands for invoking the C, C++ and Fortran compilers in each programming environment. +Finally, the new system that we defined may be identified by the hostname ``tresa`` (see the :js:attr:`hostnames` configuration parameter). +This will help ReFrame to automatically pick the right configuration when running on it. +Notice, how the ``generic`` system matches any hostname, so that it acts as a fallback system. + +.. note:: + + The different systems in the configuration file are tried in order and the first match is picked. + This practically means that the more general the selection pattern for a system is, the lower in the list of systems should be. + +The :doc:`configure` page describes the configuration file in more detail and the :doc:`config_reference` provides a complete reference guide of all the configuration options of ReFrame. + +Let's now rerun our "Hello, World!" tests: + + +.. code-block:: console + + ./bin/reframe -C tutorials/config/mysettings.py -c tutorials/basics/hello/hello2.py -r + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 986c3505) + command: './bin/reframe -C tutorials/config/mysettings.py -c tutorials/basics/hello/hello2.py -r' + launched by: user@tresa.local + working directory: '/Users/user/Repositories/reframe' + settings file: 'tutorials/config/mysettings.py' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/hello/hello2.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' + + [==========] Running 2 check(s) + [==========] Started on Sun Jun 21 19:36:22 2020 + + [----------] started processing HelloMultiLangTest_c (HelloMultiLangTest_c) + [ RUN ] HelloMultiLangTest_c on catalina:default using gnu + [ RUN ] HelloMultiLangTest_c on catalina:default using clang + [----------] finished processing HelloMultiLangTest_c (HelloMultiLangTest_c) + + [----------] started processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + [ RUN ] HelloMultiLangTest_cpp on catalina:default using gnu + [ RUN ] HelloMultiLangTest_cpp on catalina:default using clang + [----------] finished processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + + [----------] waiting for spawned checks to finish + [ OK ] (1/4) HelloMultiLangTest_cpp on catalina:default using gnu [compile: 0.768s run: 1.131s total: 1.928s] + [ OK ] (2/4) HelloMultiLangTest_c on catalina:default using gnu [compile: 0.509s run: 2.194s total: 2.763s] + [ OK ] (3/4) HelloMultiLangTest_c on catalina:default using clang [compile: 0.255s run: 2.059s total: 2.345s] + [ OK ] (4/4) HelloMultiLangTest_cpp on catalina:default using clang [compile: 1.068s run: 0.236s total: 1.332s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 4 test case(s) from 2 check(s) (0 failure(s)) + [==========] Finished on Sun Jun 21 19:36:25 2020 + + +Notice how the same tests are now tried with both the ``gnu`` and ``clang`` programming environments, without having to touch them at all! +That's one of the powerful features of ReFrame and we shall see later on, how easily we can port our tests to an HPC cluster with minimal changes. +In order to instruct ReFrame to use our configuration file, we use the ``-C`` command line option. +Since we don't want to type it throughout the tutorial, we will now set it in the environment: + +.. code-block:: console + + export RFM_CONFIG_FILE=$(pwd)/tutorials/config/mysettings.py + + +A Multithreaded "Hello, World!" +------------------------------- + +We extend our C++ "Hello, World!" example to print the greetings from multiple threads: + + +.. literalinclude:: ../tutorials/basics/hellomp/src/hello_threads.cpp + :lines: 6- + +This program takes as argument the number of threads it will create and it uses ``std::thread``, which is C++11 addition, meaning that we will need to pass ``-std=c++11`` to our compilers. +Here is the corresponding ReFrame test, where the new concepts introduced are highlighted: + +.. literalinclude:: ../tutorials/basics/hellomp/hellomp1.py + :lines: 6- + :emphasize-lines: 11-13 + +ReFrame delegates the compilation of a test to a *build system*, which is an abstraction of the steps needed to compile the test. +Build system take also care of interactions with the programming environment if necessary. +Compilation flags are a property of the build system. +If not explicitly specified, ReFrame will try to pick the correct build system (e.g., CMake, Autotools etc.) by inspecting the test resources, but in cases as the one presented here where we need to set the compilation flags, we need to specify a build system explicitly. +In this example, we instruct ReFrame to compile a single source file using the ``-std=c++11 -Wall`` compilation flags. +Finally, we set the arguments to be passed to the generated executable in :attr:`executable_opts `. + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 986c3505) + command: './bin/reframe -c tutorials/basics/hellomp/hellomp1.py -r' + launched by: user@tresa.local + working directory: '/Users/user/Repositories/reframe' + settings file: '/Users/user/Repositories/reframe/tutorials/config/settings.py' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/hellomp/hellomp1.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Mon Jun 22 00:58:27 2020 + + [----------] started processing HelloThreadedTest (HelloThreadedTest) + [ RUN ] HelloThreadedTest on catalina:default using gnu + [ RUN ] HelloThreadedTest on catalina:default using clang + [----------] finished processing HelloThreadedTest (HelloThreadedTest) + + [----------] waiting for spawned checks to finish + [ OK ] (1/2) HelloThreadedTest on catalina:default using gnu [compile: 1.354s run: 1.250s total: 2.639s] + [ OK ] (2/2) HelloThreadedTest on catalina:default using clang [compile: 1.202s run: 0.238s total: 1.468s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 2 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Mon Jun 22 00:58:29 2020 + + +Everything looks fine, but let's inspect the actual output of one of the tests: + + +.. code-block:: console + + cat output/catalina/default/clang/HelloThreadedTest/rfm_HelloThreadedTest_job.out + + +.. code-block:: none + + [[[[ 8] Hello, World! + 1] Hello, World! + 5[[0[ 7] Hello, World! + ] ] Hello, World! + [ Hello, World! + 6[] Hello, World! + 9] Hello, World! + 2 ] Hello, World! + 4] [[10 3] Hello, World! + ] Hello, World! + [Hello, World! + 11] Hello, World! + [12] Hello, World! + [13] Hello, World! + [14] Hello, World! + [15] Hello, World! + + +Not exactly what we were looking for! +In the following we write a more robust sanity check that can catch this havoc. + + +----------------------------- +More advanced sanity checking +----------------------------- + +Sanity checking of a test's outcome is quite powerful in ReFrame. +So far, we have seen only a ``grep``-like search for a string in the output, but ReFrame's ``sanity_patterns`` are much more capable than this. +In fact, you can practically do almost any operation in the output and process it as you would like before assessing the test's sanity. +The syntax feels also quite natural since it is fully integrated in Python. + +In the following we extend the sanity checking of the multithreaded "Hello, World!", such that not only the output pattern we are looking for is more restrictive, but also we check that all the threads produce a greetings line. + +.. literalinclude:: ../tutorials/basics/hellomp/hellomp2.py + :lines: 6- + :emphasize-lines: 14-16 + +The sanity checking is straightforward. +We find all the matches of the required pattern, we count them and finally we check their number. +Both statements here are lazily evaluated. +They will not be executed where they appear, but rather at the sanity checking phase. +ReFrame provides lazily evaluated counterparts for most of the builtin Python functions, such the :func:`len` function here. +Also whole expressions can be lazily evaluated if one of the operands is deferred, as is the case in this example with the assignment to ``num_messages``. +This makes the sanity checking mechanism quite powerful and straightforward to reason about, without having to rely on complex pattern matching techniques. +:doc:`sanity_functions_reference` provides a complete reference of the sanity functions provided by ReFrame, but users can also define their own, as described in :doc:`deferrables`. + + +Let's run this version of the test now and see if it fails: + +.. code-block:: console + + ./bin/reframe -c tutorials/basics/hellomp/hellomp2.py -r + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: e64355a3) + command: './bin/reframe -c tutorials/basics/hellomp/hellomp2.py -r' + launched by: user@tresa.local + working directory: '/Users/user/Repositories/reframe' + settings file: '/Users/user/Repositories/reframe/tutorials/config/settings.py' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/hellomp/hellomp2.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Mon Jun 22 20:53:02 2020 + + [----------] started processing HelloThreadedExtendedTest (HelloThreadedExtendedTest) + [ RUN ] HelloThreadedExtendedTest on catalina:default using gnu + [ RUN ] HelloThreadedExtendedTest on catalina:default using clang + [----------] finished processing HelloThreadedExtendedTest (HelloThreadedExtendedTest) + + [----------] waiting for spawned checks to finish + [ FAIL ] (1/2) HelloThreadedExtendedTest on catalina:default using gnu [compile: 1.003s run: 0.839s total: 1.871s] + [ FAIL ] (2/2) HelloThreadedExtendedTest on catalina:default using clang [compile: 0.790s run: 0.141s total: 0.954s] + [----------] all spawned checks have finished + + [ FAILED ] Ran 2 test case(s) from 1 check(s) (2 failure(s)) + [==========] Finished on Mon Jun 22 20:53:04 2020 + + ============================================================================== + SUMMARY OF FAILURES + ------------------------------------------------------------------------------ + FAILURE INFO for HelloThreadedExtendedTest + * Test Description: HelloThreadedExtendedTest + * System partition: catalina:default + * Environment: gnu + * Stage directory: /Users/user/Repositories/reframe/stage/catalina/default/gnu/HelloThreadedExtendedTest + * Node list: tresa.local + * Job type: local (id=36805) + * Maintainers: [] + * Failing phase: sanity + * Rerun with '-n HelloThreadedExtendedTest -p gnu --system catalina:default' + * Reason: sanity error: 13 != 16 + ------------------------------------------------------------------------------ + FAILURE INFO for HelloThreadedExtendedTest + * Test Description: HelloThreadedExtendedTest + * System partition: catalina:default + * Environment: clang + * Stage directory: /Users/user/Repositories/reframe/stage/catalina/default/clang/HelloThreadedExtendedTest + * Node list: tresa.local + * Job type: local (id=36815) + * Maintainers: [] + * Failing phase: sanity + * Rerun with '-n HelloThreadedExtendedTest -p clang --system catalina:default' + * Reason: sanity error: 12 != 16 + ------------------------------------------------------------------------------ + + +As expected, only some of lines are printed correctly which makes the test fail. +To fix this test, we need to compile with ``-DSYNC_MESSAGES``, which will synchronize the printing of messages. + +.. literalinclude:: ../tutorials/basics/hellomp/hellomp3.py + :lines: 6- + :emphasize-lines: 13 + + +Writing A Performance Test +-------------------------- + +An important aspect of regression testing is checking for performance regressions. +In this example, we will write a test that downloads the `STREAM `__ benchmark, compiles it, runs it and records its performance. +In the test below, we highlight the lines that introduce new concepts. + +.. literalinclude:: ../tutorials/basics/stream/stream1.py + :lines: 6- + :emphasize-lines: 10-12,17-20,23-32 + +First of all, notice that we restrict the programming environments to ``gnu`` only, since this test requires OpenMP, which our installation of Clang does not have. +The next thing to notice is the :attr:`prebuild_cmds ` attribute, which provides a list of commands to be executed before the build step. +These commands will be executed from the test's stage directory. +In this case, we just fetch the source code of the benchmark. +For running the benchmark, we need to set the OpenMP number of threads and pin them to the right CPUs through the ``OMP_NUM_THREADS`` and ``OMP_PLACES`` environment variables. +You can set environment variables in a ReFrame test through the :attr:`variables ` dictionary. + +What makes a ReFrame test a performance test is the definition of the :attr:`perf_patterns ` attribute. +This is a dictionary where the keys are *performance variables* and the values are lazily evaluated expressions for extracting the performance variable values from the test's output. +In this example, we extract four performance variables, namely the memory bandwidth values for each of the "Copy", "Scale", "Add" and "Triad" sub-benchmarks of STREAM and we do so by using the :func:`extractsingle ` sanity function. +For each of the sub-benchmarks we extract the "Best Rate MB/s" column of the output (see below) and wee convert that to a float. + +.. code-block:: none + + Function Best Rate MB/s Avg time Min time Max time + Copy: 24939.4 0.021905 0.021527 0.022382 + Scale: 16956.3 0.031957 0.031662 0.032379 + Add: 18648.2 0.044277 0.043184 0.046349 + Triad: 19133.4 0.042935 0.042089 0.044283 + + +Let's run the test now: + + +.. code-block:: console + + ./bin/reframe -c tutorials/basics/stream/stream1.py -r --performance-report + +The :option:`--performance-report` will generated a short report at the end for each performance test that has run. + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 9d92d0ec) + command: './bin/reframe -c tutorials/basics/stream/stream.py -r --performance-report' + launched by: user@tresa.local + working directory: '/Users/user/Repositories/reframe' + settings file: '/Users/user/Repositories/reframe/tutorials/config/settings.py' + check search path: '/Users/user/Repositories/reframe/tutorials/basics/stream/stream.py' + stage directory: '/Users/user/Repositories/reframe/stage' + output directory: '/Users/user/Repositories/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Wed Jun 24 00:17:59 2020 + + [----------] started processing StreamTest (StreamTest) + [ RUN ] StreamTest on catalina:default using gnu + [----------] finished processing StreamTest (StreamTest) + + [----------] waiting for spawned checks to finish + [ OK ] (1/1) StreamTest on catalina:default using gnu [compile: 3.466s run: 2.283s total: 5.795s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 1 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Wed Jun 24 00:18:05 2020 + ============================================================================== + PERFORMANCE REPORT + ------------------------------------------------------------------------------ + StreamTest + - catalina:default + - gnu + * num_tasks: 1 + * Copy: 25238.5 (no unit specified) + * Scale: 16837.3 (no unit specified) + * Add: 18431.8 (no unit specified) + * Triad: 18833.1 (no unit specified) + ------------------------------------------------------------------------------ + + +----------------------- +Adding reference values +----------------------- + +A performance test would not be so meaningful, if we couldn't test the obtained performance against a reference value. +ReFrame offers the possibility to set references for each of the performance variables defined in a test and also set different references for different systems. +In the following example, we set the reference values for all the STREAM sub-benchmarks for the system we are currently running on. + +.. note:: + + Optimizing STREAM benchmark performance is outside the scope of this tutorial. + + +.. literalinclude:: ../tutorials/basics/stream/stream2.py + :lines: 6- + :emphasize-lines: 33- + + +The performance reference tuple consists of the reference value, the lower and upper thresholds expressed as fractional numbers relative to the reference value, and the unit of measurement. +If any of the thresholds is not relevant, :class:`None` may be used instead. + +If any obtained performance value is beyond its respective thresholds, the test will fail with a summary as shown below: + +.. code-block:: none + + FAILURE INFO for StreamWithRefTest + * Test Description: StreamWithRefTest + * System partition: catalina:default + * Environment: gnu + * Stage directory: /Users/user/Repositories/reframe/stage/catalina/default/gnu/StreamWithRefTest + * Node list: tresa.local + * Job type: local (id=62114) + * Maintainers: [] + * Failing phase: performance + * Rerun with '-n StreamWithRefTest -p gnu --system catalina:default' + * Reason: performance error: failed to meet reference: Copy=24586.5, expected 55200 (l=52440.0, u=57960.0) + + + +------------------------------ +Examining the performance logs +------------------------------ + +ReFrame has a powerful mechanism for logging its activities as well as performance data. +It supports different types of log channels and it can send data simultaneously in any number of them. +For examples, performance data might be logged in files and the same time being send to Syslog or to a centralized log management server. +By default (i.e., starting off from the builtin configuration file), ReFrame sends performance data to files per test under the ``perflogs/`` directory: + +.. code-block:: none + + perflogs + └── catalina + └── default + ├── StreamTest.log + └── StreamWithRefTest.log + +ReFrame creates a log file per test per system and per partition and appends to it every time the test is run on that system/partition combination. +Let's inspect the log file from our last test: + +.. code-block:: console + + tail perflogs/catalina/default/StreamWithRefTest.log + + +.. code-block:: none + + 2020-06-24T00:27:06|reframe 3.1-dev0 (rev: 9d92d0ec)|StreamWithRefTest on catalina:default using gnu|jobid=58384|Copy=24762.2|ref=25200 (l=-0.05, u=0.05)|MB/s + 2020-06-24T00:27:06|reframe 3.1-dev0 (rev: 9d92d0ec)|StreamWithRefTest on catalina:default using gnu|jobid=58384|Scale=16784.6|ref=16800 (l=-0.05, u=0.05)|MB/s + 2020-06-24T00:27:06|reframe 3.1-dev0 (rev: 9d92d0ec)|StreamWithRefTest on catalina:default using gnu|jobid=58384|Add=18553.8|ref=18500 (l=-0.05, u=0.05)|MB/s + 2020-06-24T00:27:06|reframe 3.1-dev0 (rev: 9d92d0ec)|StreamWithRefTest on catalina:default using gnu|jobid=58384|Triad=18679.0|ref=18800 (l=-0.05, u=0.05)|MB/s + 2020-06-24T12:42:07|reframe 3.1-dev0 (rev: 138cbd68)|StreamWithRefTest on catalina:default using gnu|jobid=62114|Copy=24586.5|ref=55200 (l=-0.05, u=0.05)|MB/s + 2020-06-24T12:42:07|reframe 3.1-dev0 (rev: 138cbd68)|StreamWithRefTest on catalina:default using gnu|jobid=62114|Scale=16880.6|ref=16800 (l=-0.05, u=0.05)|MB/s + 2020-06-24T12:42:07|reframe 3.1-dev0 (rev: 138cbd68)|StreamWithRefTest on catalina:default using gnu|jobid=62114|Add=18570.4|ref=18500 (l=-0.05, u=0.05)|MB/s + 2020-06-24T12:42:07|reframe 3.1-dev0 (rev: 138cbd68)|StreamWithRefTest on catalina:default using gnu|jobid=62114|Triad=19048.3|ref=18800 (l=-0.05, u=0.05)|MB/s + +Several information are printed for each run, such as the performance variables, their value, their references and thresholds etc. +The default format is in a form suitable for easy parsing, but you may fully control not only the format, but also what is being logged from the configuration file. +:doc:`configure` and :doc:`config_reference` cover logging in ReFrame in much more detail. + + + +Porting The Tests to an HPC cluster +----------------------------------- + +It's now time to port our tests to an HPC cluster. +Obviously, HPC clusters are much more complex than our laptop or PC. +Usually there are many more compilers, the user environment is handled in a different way, and the way to launch the tests varies significantly, since you have to go through a workload manager in order to acces the actual compute nodes. +Besides that, there might be multiple types of compute nodes that we would like to run our tests on, but each type might be accessed in a different way. +It is already apparent that porting even an as simple as a "Hello, World" test to such a system is not that straightforward. +As we shall see in this section, ReFrame makes that pretty easy. + +-------------------------- +Adapting the configuration +-------------------------- + +Our target system is the `Piz Daint `__ supercomputer at CSCS, but you can adapt the process to your target HPC system. +In ReFrame, all the details of the various interactions of a test with the system environment are handled transparently and are set up in its configuration file. +Let's extend our configuration file for Piz Daint. + + +.. literalinclude:: ../tutorials/config/settings.py + :lines: 10- + :emphasize-lines: 17-50,72-103 + + +First of all, we need to define a new system and set the list of hostnames that will help ReFrame identify it. +We also set the :js:attr:`modules_system` configuration parameter to instruct ReFrame that this system makes use of the `environment modules `__ for managing the user environment. +Then we define the system partitions that we want to test. +In this case, we define three partitions: + +1. the login nodes, +2. the multicore partition (2x Broadwell CPUs per node) and +3. the hybrid partition (1x Haswell CPU + 1x Pascal GPU). + +.. |srun| replace:: :obj:`srun` +.. _srun: https://slurm.schedmd.com/srun.html + +The login nodes are pretty much similar to the ``catalina:default`` partition which corresponded to our laptop: tests will be launched and run locally. +The other two partitions are handled by `Slurm `__ and parallel jobs are launched using the |srun|_ command. +Additionally, in order to access the different types of nodes represented by those partitions, users have to specify either ``-C mc`` or ``-C gpu`` options along with their account. +This is what we do exactly with the :js:attr:`access` partition configuration option. + +.. note:: + + System partitions in ReFrame do not necessarily correspond to real job scheduler partitions. + +Piz Daint's programming environment offers four compilers: Cray, GNU, Intel and PGI. +We want to test all of them, so we include them in the :js:attr:`environs` lists. +Notice that we do not include Clang in the list, since there is no such compiler on this particular system. + +Before looking into the definition of the new environments for the four compilers, it is worth mentioning the :js:attr:`max_jobs` parameter. +This parameter specifies the maximum number of ReFrame test jobs that can be simultaneously in flight. +ReFrame will try to keep concurrency close to this limit (but not exceeding it). +By default, this is set to one, so you are advised to set it to a higher number if you want to increase the throughput of completed tests. + +The new environments are defined similarly to the ones we had for our local system, except that now we set two more parameters: the :js:attr:`modules` and the :js:attr:`target_systems`. +The :js:attr:`modules` parameter is a list of environment modules that needs to be loaded, in order to make available this compiler. +The :js:attr:`target_systems` parameter restricts the environment definition to a list of specific systems or system partitions. +This allows us to redefine environments for different systems, as for example the ``gnu`` environment in this case. +ReFrame will always pick the definition that is a closest match for the current system. +In this example, it will pick the second definition for ``gnu`` whenever it runs on the system named ``daint``, and the first in every other occasion. + +----------------- +Running the tests +----------------- + +We are now ready to run our tests on Piz Daint. +We will only do so with the final versions of the tests from the previous section, which we will select using :option:`-n` option. + +.. code-block:: console + + ./bin/reframe -C tutorials/config/settings.py -c tutorials/basics/ -R -n 'HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest' --performance-report -r + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: 6e3204a7) + command: './bin/reframe -C tutorials/config/settings.py -c tutorials/basics/ -R -n HelloMultiLangTest|HelloThreadedExtended2Test|StreamWithRefTest --performance-report -r' + launched by: user@daint101 + working directory: '/users/user/Devel/reframe' + settings file: 'tutorials/config/settings.py' + check search path: (R) '/users/user/Devel/reframe/tutorials/basics' + stage directory: '/users/user/Devel/reframe/stage' + output directory: '/users/user/Devel/reframe/output' + + [==========] Running 4 check(s) + [==========] Started on Thu Jun 25 19:48:41 2020 + + [----------] started processing HelloMultiLangTest_c (HelloMultiLangTest_c) + [ RUN ] HelloMultiLangTest_c on daint:login using gnu + [ RUN ] HelloMultiLangTest_c on daint:login using intel + [ RUN ] HelloMultiLangTest_c on daint:login using pgi + [ RUN ] HelloMultiLangTest_c on daint:login using cray + [ RUN ] HelloMultiLangTest_c on daint:gpu using gnu + [ RUN ] HelloMultiLangTest_c on daint:gpu using intel + [ RUN ] HelloMultiLangTest_c on daint:gpu using pgi + [ RUN ] HelloMultiLangTest_c on daint:gpu using cray + [ RUN ] HelloMultiLangTest_c on daint:mc using gnu + [ RUN ] HelloMultiLangTest_c on daint:mc using intel + [ RUN ] HelloMultiLangTest_c on daint:mc using pgi + [ RUN ] HelloMultiLangTest_c on daint:mc using cray + [----------] finished processing HelloMultiLangTest_c (HelloMultiLangTest_c) + + [----------] started processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + [ RUN ] HelloMultiLangTest_cpp on daint:login using gnu + [ RUN ] HelloMultiLangTest_cpp on daint:login using intel + [ RUN ] HelloMultiLangTest_cpp on daint:login using pgi + [ RUN ] HelloMultiLangTest_cpp on daint:login using cray + [ RUN ] HelloMultiLangTest_cpp on daint:gpu using gnu + [ RUN ] HelloMultiLangTest_cpp on daint:gpu using intel + [ RUN ] HelloMultiLangTest_cpp on daint:gpu using pgi + [ RUN ] HelloMultiLangTest_cpp on daint:gpu using cray + [ RUN ] HelloMultiLangTest_cpp on daint:mc using gnu + [ RUN ] HelloMultiLangTest_cpp on daint:mc using intel + [ RUN ] HelloMultiLangTest_cpp on daint:mc using pgi + [ RUN ] HelloMultiLangTest_cpp on daint:mc using cray + [----------] finished processing HelloMultiLangTest_cpp (HelloMultiLangTest_cpp) + + [----------] started processing HelloThreadedExtended2Test (HelloThreadedExtended2Test) + [ RUN ] HelloThreadedExtended2Test on daint:login using gnu + [ RUN ] HelloThreadedExtended2Test on daint:login using intel + [ RUN ] HelloThreadedExtended2Test on daint:login using pgi + [ RUN ] HelloThreadedExtended2Test on daint:login using cray + [ RUN ] HelloThreadedExtended2Test on daint:gpu using gnu + [ RUN ] HelloThreadedExtended2Test on daint:gpu using intel + [ RUN ] HelloThreadedExtended2Test on daint:gpu using pgi + [ RUN ] HelloThreadedExtended2Test on daint:gpu using cray + [ RUN ] HelloThreadedExtended2Test on daint:mc using gnu + [ RUN ] HelloThreadedExtended2Test on daint:mc using intel + [ RUN ] HelloThreadedExtended2Test on daint:mc using pgi + [ RUN ] HelloThreadedExtended2Test on daint:mc using cray + [----------] finished processing HelloThreadedExtended2Test (HelloThreadedExtended2Test) + + [----------] started processing StreamWithRefTest (StreamWithRefTest) + [ RUN ] StreamWithRefTest on daint:login using gnu + [ RUN ] StreamWithRefTest on daint:gpu using gnu + [ RUN ] StreamWithRefTest on daint:mc using gnu + [----------] finished processing StreamWithRefTest (StreamWithRefTest) + + [----------] waiting for spawned checks to finish + [ OK ] ( 1/39) StreamWithRefTest on daint:login using gnu [compile: 2.516s run: 8.253s total: 10.904s] + [ OK ] ( 2/39) HelloThreadedExtended2Test on daint:gpu using intel [compile: 2.402s run: 26.498s total: 29.573s] + [ OK ] ( 3/39) HelloThreadedExtended2Test on daint:login using cray [compile: 0.936s run: 31.749s total: 33.515s] + [ OK ] ( 4/39) HelloThreadedExtended2Test on daint:login using intel [compile: 2.484s run: 38.162s total: 41.500s] + [ OK ] ( 5/39) HelloMultiLangTest_cpp on daint:mc using pgi [compile: 2.083s run: 45.088s total: 48.052s] + [ OK ] ( 6/39) HelloMultiLangTest_cpp on daint:mc using gnu [compile: 1.906s run: 50.757s total: 53.713s] + [ OK ] ( 7/39) HelloMultiLangTest_cpp on daint:gpu using intel [compile: 2.138s run: 57.063s total: 60.459s] + [ OK ] ( 8/39) HelloMultiLangTest_cpp on daint:login using intel [compile: 2.138s run: 66.385s total: 69.937s] + [ OK ] ( 9/39) HelloMultiLangTest_c on daint:mc using intel [compile: 1.900s run: 75.088s total: 78.428s] + [ OK ] (10/39) HelloMultiLangTest_c on daint:gpu using intel [compile: 1.903s run: 82.938s total: 86.443s] + [ OK ] (11/39) HelloMultiLangTest_c on daint:login using intel [compile: 1.911s run: 90.911s total: 94.586s] + [ OK ] (12/39) HelloThreadedExtended2Test on daint:login using gnu [compile: 2.181s run: 5.360s total: 44.519s] + [ OK ] (13/39) HelloMultiLangTest_cpp on daint:gpu using pgi [compile: 2.100s run: 17.950s total: 57.466s] + [ OK ] (14/39) HelloMultiLangTest_cpp on daint:gpu using gnu [compile: 2.148s run: 23.833s total: 63.556s] + [ OK ] (15/39) HelloMultiLangTest_cpp on daint:login using pgi [compile: 2.123s run: 27.244s total: 67.101s] + [ OK ] (16/39) HelloMultiLangTest_cpp on daint:login using gnu [compile: 1.925s run: 33.013s total: 72.699s] + [ OK ] (17/39) HelloMultiLangTest_c on daint:mc using pgi [compile: 1.760s run: 36.179s total: 75.724s] + [ OK ] (18/39) HelloMultiLangTest_c on daint:mc using gnu [compile: 1.643s run: 41.386s total: 80.980s] + [ OK ] (19/39) HelloMultiLangTest_c on daint:gpu using pgi [compile: 1.618s run: 44.076s total: 83.805s] + [ OK ] (20/39) HelloMultiLangTest_c on daint:gpu using gnu [compile: 1.784s run: 49.160s total: 89.222s] + [ OK ] (21/39) HelloMultiLangTest_c on daint:login using pgi [compile: 1.676s run: 51.922s total: 92.032s] + [ OK ] (22/39) HelloMultiLangTest_c on daint:login using gnu [compile: 1.747s run: 56.999s total: 97.205s] + [ OK ] (23/39) HelloThreadedExtended2Test on daint:mc using pgi [compile: 2.802s run: 16.336s total: 19.372s] + [ OK ] (24/39) HelloThreadedExtended2Test on daint:mc using gnu [compile: 2.146s run: 23.128s total: 25.670s] + [ OK ] (25/39) HelloThreadedExtended2Test on daint:gpu using gnu [compile: 2.165s run: 33.585s total: 36.414s] + [ OK ] (26/39) HelloMultiLangTest_cpp on daint:mc using cray [compile: 0.624s run: 47.468s total: 49.001s] + [ OK ] (27/39) HelloMultiLangTest_cpp on daint:gpu using cray [compile: 0.635s run: 56.551s total: 58.307s] + [ OK ] (28/39) HelloMultiLangTest_c on daint:mc using cray [compile: 0.328s run: 75.253s total: 76.864s] + [ OK ] (29/39) HelloMultiLangTest_c on daint:login using cray [compile: 0.374s run: 91.505s total: 93.322s] + [ OK ] (30/39) HelloThreadedExtended2Test on daint:mc using intel [compile: 2.458s run: 22.705s total: 25.399s] + [ OK ] (31/39) HelloThreadedExtended2Test on daint:gpu using pgi [compile: 2.715s run: 29.752s total: 32.867s] + [ OK ] (32/39) HelloMultiLangTest_cpp on daint:mc using intel [compile: 2.097s run: 54.858s total: 57.513s] + [ OK ] (33/39) HelloMultiLangTest_c on daint:gpu using cray [compile: 0.319s run: 86.715s total: 87.750s] + [ OK ] (34/39) HelloMultiLangTest_cpp on daint:login using cray [compile: 0.637s run: 71.388s total: 72.518s] + [ OK ] (35/39) HelloThreadedExtended2Test on daint:login using pgi [compile: 2.615s run: 43.999s total: 48.315s] + [ OK ] (36/39) StreamWithRefTest on daint:gpu using gnu [compile: 2.274s run: 17.018s total: 19.336s] + [ OK ] (37/39) HelloThreadedExtended2Test on daint:gpu using cray [compile: 0.917s run: 33.426s total: 34.523s] + [ OK ] (38/39) StreamWithRefTest on daint:mc using gnu [compile: 2.129s run: 16.200s total: 18.366s] + [ OK ] (39/39) HelloThreadedExtended2Test on daint:mc using cray [compile: 0.911s run: 52.870s total: 53.815s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 39 test case(s) from 4 check(s) (0 failure(s)) + [==========] Finished on Thu Jun 25 19:51:00 2020 + ============================================================================== + PERFORMANCE REPORT + ------------------------------------------------------------------------------ + StreamWithRefTest + - daint:login + - gnu + * num_tasks: 1 + * Copy: 72638.7 MB/s + * Scale: 45172.4 MB/s + * Add: 49001.9 MB/s + * Triad: 48925.2 MB/s + - daint:gpu + - gnu + * num_tasks: 1 + * Copy: 50525.0 MB/s + * Scale: 34746.8 MB/s + * Add: 38144.5 MB/s + * Triad: 38459.9 MB/s + - daint:mc + - gnu + * num_tasks: 1 + * Copy: 18931.9 MB/s + * Scale: 10460.8 MB/s + * Add: 11032.2 MB/s + * Triad: 11024.0 MB/s + ------------------------------------------------------------------------------ + + +There it is! +Without any change in our tests, we could simply run them in a HPC cluster with all of its intricacies. +Notice how our just four tests expanded to almost 40 test cases on that particular HPC cluster! +One reason we could run immediately our tests on a new system was that we have not been restricting neither the valid system they can run nor the valid programming environments they can run with (except for the STREAM test). +Otherwise we would have to add ``daint`` and its corresponding programming environments in :attr:`valid_systems` and :attr:`valid_prog_environs` lists respectively. + +.. tip:: + + A quick way to try a test on a new system, if it's not generic, is to pass the :option:`--skip-system-check` and the :option:`--skip-prgenv-check` command line options which will cause ReFrame to skip any test validity checks for systems or programming environments. + +Although the tests remain the same, ReFrame has generated completely different job scripts for each test depending on where it was going to run. +Let's check the job script generated for the ``StreamWithRefTest``: + +.. code-block:: console + + cat output/daint/gpu/gnu/StreamWithRefTest/rfm_StreamWithRefTest_job.sh + +.. code-block:: bash + + #!/bin/bash + #SBATCH --job-name="rfm_StreamWithRefTest_job" + #SBATCH --ntasks=1 + #SBATCH --output=rfm_StreamWithRefTest_job.out + #SBATCH --error=rfm_StreamWithRefTest_job.err + #SBATCH --time=0:10:0 + #SBATCH -A csstaff + #SBATCH --constraint=gpu + module unload PrgEnv-cray + module load PrgEnv-gnu + export OMP_NUM_THREADS=4 + export OMP_PLACES=cores + srun ./StreamWithRefTest + +Whereas the exact same test running on our laptop was as simple as the following: + +.. code-block:: bash + + #!/bin/bash + export OMP_NUM_THREADS=4 + export OMP_PLACES=cores + ./StreamWithRefTest + +In ReFrame, you don't have to care about all the system interaction details, but rather about the logic of your tests as we shall see in the next section. + + +----------------------------------------------------------- +Adapting a test to new systems and programming environments +----------------------------------------------------------- + +Unless a test is rather generic, you will need to do some adaptations for the system that you port it to. +In this case, we will adapt the STREAM benchmark so as to run it with multiple compiler and adjust its execution parameters based on the target architecture of each partition. +Let's see and comment the changes: + +.. literalinclude:: ../tutorials/basics/stream/stream3.py + :lines: 6- + :emphasize-lines: 9,37- + +First of all, we need to add the new programming environments in the list of the supported ones. +Now there is the problem that each compiler has its own flags for enabling OpenMP, so we need to differentiate the behavior of the test based on the programming environment. +For this reason, we define the flags for each compiler in a separate dictionary (``self.flags``) and we set them in the :func:`setflags` pipeline hook. +Let's explain what is this all about. +When ReFrame loads a test file, it instantiates all the tests it finds in it. +Based on the system ReFrame runs on and the supported environments of the tests, it will generate different test cases for each system partition and environment combination and it will finally send the test cases for execution. +During its execution, a test case goes through the *regression test pipeline*, which is a series of well defined phases. +Users can attach arbitrary functions to run before or after any pipeline stage and this is exactly what the :func:`setflags` function is. +We instruct ReFrame to run this function before the test enters the ``compile`` stage and set accordingly the compilation flags. +The system partition and the programming environment of the currently running test case are available to a ReFrame test through the :attr:`current_partition ` and :attr:`current_environ ` attributes respectively. +These attributes, however, are only set after the first stage (``setup``) of the pipeline is executed, so we can't use them inside the test's constructor. + +We do exactly the same for setting the ``OMP_NUM_THREADS`` environment variables depending on the system partition we are running on, by attaching the :func:`set_num_threads` pipeline hook to the ``run`` phase of the test. +In that same hook we also set the :attr:`num_cpus_per_task ` attribute of the test, so as to instruct the backend job scheduler to properly assign CPU cores to the test. +In ReFrame tests you can set a series of task allocation attributes that will be used by the backend schedulers to emit the right job submission script. +The section :ref:`scheduler_options` of the :doc:`regression_test_api` summarizes these attributes and the actual backend scheduler options that they correspond to. + +For more information about the regression test pipeline and how ReFrame executes the tests in general, have a look at :doc:`pipeline`. + +.. note:: + + ReFrame tests are ordinary Python classes so you can define your own attributes as we do with :attr:`flags` and :attr:`cores` in this example. + +Let's run our adapted test now: + +.. code-block:: console + + ./bin/reframe -C tutorials/config/settings.py -c tutorials/basics/stream/stream3.py -r --performance-report + + +.. code-block:: none + + [ReFrame Setup] + version: 3.1-dev0 (rev: cf4efce5) + command: './bin/reframe -C tutorials/config/settings.py -c tutorials/basics/stream/stream3.py -r --performance-report' + launched by: user@daint101 + working directory: '/users/user/Devel/reframe' + settings file: 'tutorials/config/settings.py' + check search path: '/users/user/Devel/reframe/tutorials/basics/stream/stream3.py' + stage directory: '/users/user/Devel/reframe/stage' + output directory: '/users/user/Devel/reframe/output' + + [==========] Running 1 check(s) + [==========] Started on Sat Jun 27 09:25:08 2020 + + [----------] started processing StreamMultiSysTest (StreamMultiSysTest) + [ RUN ] StreamMultiSysTest on daint:login using gnu + [ RUN ] StreamMultiSysTest on daint:login using intel + [ RUN ] StreamMultiSysTest on daint:login using pgi + [ RUN ] StreamMultiSysTest on daint:login using cray + [ RUN ] StreamMultiSysTest on daint:gpu using gnu + [ RUN ] StreamMultiSysTest on daint:gpu using intel + [ RUN ] StreamMultiSysTest on daint:gpu using pgi + [ RUN ] StreamMultiSysTest on daint:gpu using cray + [ RUN ] StreamMultiSysTest on daint:mc using gnu + [ RUN ] StreamMultiSysTest on daint:mc using intel + [ RUN ] StreamMultiSysTest on daint:mc using pgi + [ RUN ] StreamMultiSysTest on daint:mc using cray + [----------] finished processing StreamMultiSysTest (StreamMultiSysTest) + + [----------] waiting for spawned checks to finish + [ OK ] ( 1/12) StreamMultiSysTest on daint:mc using gnu [compile: 2.089s run: 8.441s total: 10.824s] + [ OK ] ( 2/12) StreamMultiSysTest on daint:gpu using pgi [compile: 2.174s run: 12.136s total: 14.812s] + [ OK ] ( 3/12) StreamMultiSysTest on daint:gpu using gnu [compile: 2.272s run: 18.251s total: 21.192s] + [ OK ] ( 4/12) StreamMultiSysTest on daint:login using pgi [compile: 2.317s run: 22.250s total: 25.389s] + [ OK ] ( 5/12) StreamMultiSysTest on daint:login using gnu [compile: 3.954s run: 28.739s total: 33.587s] + [ OK ] ( 6/12) StreamMultiSysTest on daint:mc using intel [compile: 2.382s run: 6.621s total: 9.167s] + [ OK ] ( 7/12) StreamMultiSysTest on daint:gpu using intel [compile: 2.373s run: 16.576s total: 19.265s] + [ OK ] ( 8/12) StreamMultiSysTest on daint:login using intel [compile: 2.607s run: 26.907s total: 30.021s] + [ OK ] ( 9/12) StreamMultiSysTest on daint:login using cray [compile: 1.055s run: 22.923s total: 24.242s] + [ OK ] (10/12) StreamMultiSysTest on daint:gpu using cray [compile: 0.828s run: 13.380s total: 14.379s] + [ OK ] (11/12) StreamMultiSysTest on daint:mc using pgi [compile: 2.164s run: 5.444s total: 7.661s] + [ OK ] (12/12) StreamMultiSysTest on daint:mc using cray [compile: 0.834s run: 5.281s total: 6.175s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 12 test case(s) from 1 check(s) (0 failure(s)) + [==========] Finished on Sat Jun 27 09:25:46 2020 + ============================================================================== + PERFORMANCE REPORT + ------------------------------------------------------------------------------ + StreamMultiSysTest + - daint:login + - gnu + * num_tasks: 1 + * Copy: 95919.2 MB/s + * Scale: 73725.6 MB/s + * Add: 79970.2 MB/s + * Triad: 79945.6 MB/s + - intel + * num_tasks: 1 + * Copy: 105229.2 MB/s + * Scale: 110150.2 MB/s + * Add: 115988.5 MB/s + * Triad: 115520.4 MB/s + - pgi + * num_tasks: 1 + * Copy: 99439.2 MB/s + * Scale: 73494.6 MB/s + * Add: 82817.2 MB/s + * Triad: 82274.6 MB/s + - cray + * num_tasks: 1 + * Copy: 99571.1 MB/s + * Scale: 75192.8 MB/s + * Add: 82857.8 MB/s + * Triad: 83870.1 MB/s + - daint:gpu + - gnu + * num_tasks: 1 + * Copy: 42133.8 MB/s + * Scale: 37802.8 MB/s + * Add: 43161.1 MB/s + * Triad: 43702.8 MB/s + - intel + * num_tasks: 1 + * Copy: 52103.3 MB/s + * Scale: 53698.7 MB/s + * Add: 58640.6 MB/s + * Triad: 58879.8 MB/s + - pgi + * num_tasks: 1 + * Copy: 50590.9 MB/s + * Scale: 39557.3 MB/s + * Add: 44025.2 MB/s + * Triad: 44308.2 MB/s + - cray + * num_tasks: 1 + * Copy: 50448.1 MB/s + * Scale: 38780.0 MB/s + * Add: 43289.4 MB/s + * Triad: 43485.6 MB/s + - daint:mc + - gnu + * num_tasks: 1 + * Copy: 48811.0 MB/s + * Scale: 38610.4 MB/s + * Add: 43688.6 MB/s + * Triad: 44017.7 MB/s + - intel + * num_tasks: 1 + * Copy: 52920.0 MB/s + * Scale: 49444.5 MB/s + * Add: 57869.0 MB/s + * Triad: 57948.5 MB/s + - pgi + * num_tasks: 1 + * Copy: 45228.7 MB/s + * Scale: 40545.9 MB/s + * Add: 44201.5 MB/s + * Triad: 44669.7 MB/s + - cray + * num_tasks: 1 + * Copy: 47148.2 MB/s + * Scale: 40026.3 MB/s + * Add: 44029.8 MB/s + * Triad: 44352.4 MB/s + ------------------------------------------------------------------------------ + + +Notice the improved performance of the benchmark in all partitions and the differences in performance between the different compilers. + +This concludes our introductory tutorial to ReFrame! diff --git a/docs/tutorial_misc_topics.rst b/docs/tutorial_misc_topics.rst new file mode 100644 index 0000000000..5c26f7eb9f --- /dev/null +++ b/docs/tutorial_misc_topics.rst @@ -0,0 +1,395 @@ +.. |tutorialdir_adv| replace:: :obj:`tutorial/advanced/` +.. _tutorialdir_adv: https://github.com/eth-cscs/reframe/tree/master/tutorial/advanced +.. |tutorial_adv_example1| replace:: :obj:`tutorial/advanced/src/advanced_example1.c` +.. _tutorial_adv_example1: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/advanced_example1.c +.. |limits.sh| replace:: :obj:`scripts/limits.sh` +.. _limits.sh: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/scripts/limits.sh + + + +================================== + Tutorial 2: Miscellaneous topics +================================== + +This page collects several smaller tutorials that show specific parts of ReFrame. +They all use the configuration file presented in :doc:`tutorial_basics`, which you can find in ``tutorials/config/settings.py``. +They also assumes that the reader is already familiar with the concepts presented in the basic tutorial. + + +Testing a CUDA Code +------------------- + +In this example, we will create a regression test for a simple CUDA matrix-vector multiplication kernel. + +.. literalinclude:: ../tutorials/misc/gpu/cuda.py + :lines: 6- + :emphasize-lines: 9,11,13 + +There are three new things to notice in this test. +First, we restrict the list of valid systems only to the hybrid partition of Piz Daint, since we require GPU nodes. +Second, we set the :attr:`sourcepath` to the CUDA source file as we would do with any other C, C++ or Fortran file. +ReFrame will recognize the ``.cu`` extension of the source file and it will try to invoke ``nvcc`` for compiling the code. +Finally, we define the :attr:`modules ` attribute. +This is essentially a list of environment modules that need to be loaded for running the test. +In this case and in this particular system, we need to load the ``cudatoolkit`` module, which will make available the CUDA SDK. + + +More On Building Tests +---------------------- + +We have already seen how ReFrame can compile a test with a single source file. +However, ReFrame can also build tests that use Make or a configure-Make approach. +We are going to demonstrate this through a simple C++ program that computes a dot-product of two vectors and is being compiled through a Makefile. +Additionally, we can select the type of elements for the vectors at compilation time. +Here is the C++ program: + +.. literalinclude:: ../tutorials/misc/makefiles/src/dotprod.cpp + :language: cpp + :lines: 6- + +The directory structure for this test is the following: + +.. code-block:: none + + tutorials/makefiles/ + ├── maketest.py + └── src + ├── Makefile + └── dotprod.cpp + + +Let's have a look at the test itself: + +.. literalinclude:: ../tutorials/misc/makefiles/maketest.py + :lines: 6-22 + :emphasize-lines: 11,13-14 + +First, if you're using any build system other than ``SingleSource``, you must set the :attr:`executable` attribute of the test, because ReFrame cannot know what is the actual executable to be run. +We then set the build system to :class:`Make ` and set the preprocessor flags as we would do with the :class:`SingleSource` build system. + +Let's inspect the build script generated by ReFrame: + +.. code-block:: console + + cat output/catalina/default/clang/MakefileTest_float/rfm_MakefileTest_build.sh + +.. code-block:: bash + + #!/bin/bash + + _onerror() + { + exitcode=$? + echo "-reframe: command \`$BASH_COMMAND' failed (exit code: $exitcode)" + exit $exitcode + } + + trap _onerror ERR + + make -j 1 CPPFLAGS="-DELEM_TYPE=float" + + +The compiler variables (``CC``, ``CXX`` etc.) are set based on the corresponding values specified in the `coniguration `__ of the current environment. +We can instruct the build system to ignore the default values from the environment by setting its :attr:`flags_from_environ ` attribute to false: + +.. code-block:: python + + self.build_system.flags_from_environ = False + +In this case, ``make`` will be invoked as follows: + +.. code:: + + make -j 1 CPPFLAGS="-DELEM_TYPE=float" + +Notice that the ``-j 1`` option is always generated. +We can increase the build concurrency by setting the :attr:`max_concurrency ` attribute. +Finally, we may even use a custom Makefile by setting the :attr:`Make ` attribute: + +.. code-block:: python + + self.build_system.max_concurrency = 4 + self.build_system.makefile = 'Makefile_custom' + + +As a final note, as with the :class:`SingleSource` build system, it wouldn't have been necessary to specify one in this test, if we wouldn't have to set the CPPFLAGS. +ReFrame could automatically figure out the correct build system if :attr:`sourcepath ` refers to a directory. +ReFrame will inspect the directory and it will first try to determine whether this is a CMake or Autotools-based project. +If not, as in this case, it would fall back to the :class:`Make` build system. + +More details on ReFrame's build systems can be found `here `__. + + +Retrieving the source code from a Git repository +================================================ + +It might be the case that a regression test needs to clone its source code from a remote repository. +This can be achieved in two ways with ReFrame. +One way is to set the :attr:`sourcesdir` attribute to :class:`None` and explicitly clone a repository using the :attr:`prebuild_cmds `: + +.. code-block:: python + + self.sourcesdir = None + self.prebuild_cmds = ['git clone https://github.com/me/myrepo .'] + +Alternatively, we can retrieve specifically a Git repository by assigning its URL directly to the :attr:`sourcesdir` attribute: + +.. code-block:: python + + self.sourcesdir = 'https://github.com/me/myrepo' + +ReFrame will attempt to clone this repository inside the stage directory by executing ``git clone .`` and will then procede with the build procedure as usual. + +.. note:: + ReFrame recognizes only URLs in the :attr:`sourcesdir` attribute and requires passwordless access to the repository. + This means that the SCP-style repository specification will not be accepted. + You will have to specify it as URL using the ``ssh://`` protocol (see `Git documentation page `__). + + +Adding a configuration step before compiling the code +===================================================== + +It is often the case that a configuration step is needed before compiling a code with ``make``. +To address this kind of projects, ReFrame aims to offer specific abstractions for "configure-make" style of build systems. +It supports `CMake-based `__ projects through the :class:`CMake ` build system, as well as `Autotools-based `__ projects through the :class:`Autotools ` build system. + +For other build systems, you can achieve the same effect using the :class:`Make ` build system and the :attr:`prebuild_cmds ` for performing the configuration step. +The following code snippet will configure a code with ``./custom_configure`` before invoking ``make``: + +.. code-block:: python + + self.prebuild_cmds = ['./custom_configure -with-mylib'] + self.build_system = 'Make' + self.build_system.cppflags = ['-DHAVE_FOO'] + self.build_system.flags_from_environ = False + +The generated build script will then have the following lines: + +.. code-block:: bash + + ./custom_configure -with-mylib + make -j 1 CPPFLAGS='-DHAVE_FOO' + + +Writing a Run-Only Regression Test +---------------------------------- + +There are cases when it is desirable to perform regression testing for an already built executable. +In the following test we use simply the ``echo`` Bash shell command to print a random integer between specific lower and upper bounds: +Here is the full regression test: + +.. literalinclude:: ../tutorials/misc/runonly/echorand.py + :lines: 6- + :emphasize-lines: 6 + +There is nothing special for this test compared to those presented so far except that it derives from the :class:`RunOnlyRegressionTest `. +Run-only regression tests may also have resources, as for instance a pre-compiled executable or some input data. +These resources may reside under the ``src/`` directory or under any directory specified in the :attr:`sourcesdir ` attribute. +These resources will be copied to the stage directory at the beginning of the run phase. + + +Writing a Compile-Only Regression Test +-------------------------------------- + +ReFrame provides the option to write compile-only tests which consist only of a compilation phase without a specified executable. +This kind of tests must derive from the :class:`CompileOnlyRegressionTest ` class provided by the framework. +The following test is a compile-only version of the :class:`MakefileTest` presented `previously <#more-on-building-tests>`__ which checks that no warnings are issued by the compiler: + +.. literalinclude:: ../tutorials/misc/makefiles/maketest.py + :lines: 25-33 + :emphasize-lines: 2 + +What is worth noting here is that the standard output and standard error of the test, which are accessible through the :attr:`stdout ` and :attr:`stderr ` attributes, correspond now to the standard output and error of the compilation command. +Therefore sanity checking can be done in exactly the same way as with a normal test. + + +Applying a Sanity Function Iteratively +-------------------------------------- + +It is often the case that a common sanity pattern has to be applied many times. +The following script prints 100 random integers between the limits given by the environment variables ``LOWER`` and ``UPPER``. + +.. literalinclude:: ../tutorials/misc/random/src/random_numbers.sh + :language: bash + :lines: 7- + +In the corresponding regression test we want to check that all the random numbers generated lie between the two limits, which means that a common sanity check has to be applied to all the printed random numbers. +Here is the corresponding regression test: + +.. literalinclude:: ../tutorials/misc/random/randint.py + :lines: 6- + :emphasize-lines: 12- + +First, we extract all the generated random numbers from the output. +What we want to do is to apply iteratively the :func:`assert_bounded ` sanity function for each number. +The problem here is that we cannot simply iterate over the ``numbers`` list, because that would trigger prematurely the evaluation of the :func:`extractall `. +We want to defer also the iteration. +This can be achieved by using the :func:`map() ` ReFrame sanity function, which is a replacement of Python's built-in :py:func:`map` function and does exactly what we want: it applies a function on all the elements of an iterable and returns another iterable with the transformed elements. +Passing the result of the :py:func:`map` function to the :func:`all ` sanity function ensures that all the elements lie between the desired bounds. + +There is still a small complication that needs to be addressed. +As a direct replacement of the built-in :py:func:`all` function, ReFrame's :func:`all() ` sanity function returns :class:`True` for empty iterables, which is not what we want. +So we must make sure that all 100 numbers are generated. +This is achieved by the ``sn.assert_eq(sn.count(numbers), 100)`` statement, which uses the :func:`count() ` sanity function for counting the generated numbers. +Finally, we need to combine these two conditions to a single deferred expression that will be assigned to the test's :attr:`sanity_patterns`. +As with the :keyword:`for` loop discussed above, we cannot defer the evaluation of the :keyword:`and` operator, so we use ReFrame's the :func:`and_() ` sanity function to accomplish this. + +For more information about how exactly sanity functions work and how their execution is deferred, please refer to :doc:`deferrables`. + +.. note:: + .. versionadded:: 2.13 + ReFrame offers also the :func:`allx() ` sanity function which, conversely to the builtin :func:`all()` function, will return :class:`False` if its iterable argument is empty. + + +Customizing the Generated Job Script +------------------------------------ + +It is often the case that you must run some commands before or after the parallel launch of your executable. +This can be easily achieved by using the :attr:`prerun_cmds ` and :attr:`postrun_cmds ` attributes of a ReFrame test. + +The following example is a slightly modified version of the previous one. +The lower and upper limits for the random numbers are now set inside a helper shell script in |limits.sh|_ and we want also to print the word ``FINISHED`` after our executable has finished. +In order to achieve this, we need to source the helper script just before launching the executable and ``echo`` the desired message just after it finishes. +Here is the test file: + +.. literalinclude:: ../tutorial/advanced/advanced_example7.py + +Notice the use of the :attr:`prerun_cmds` and :attr:`postrun_cmds` attributes. +These are lists of shell commands that are emitted verbatim in the job script. +The generated job script for this example is the following: + +.. code-block:: bash + + #!/bin/bash -l + #SBATCH --job-name="prerun_demo_check_daint_gpu_PrgEnv-gnu" + #SBATCH --time=0:10:0 + #SBATCH --ntasks=1 + #SBATCH --output=prerun_demo_check.out + #SBATCH --error=prerun_demo_check.err + #SBATCH --constraint=gpu + module load daint-gpu + module unload PrgEnv-cray + module load PrgEnv-gnu + source scripts/limits.sh + srun ./random_numbers.sh + echo FINISHED + +ReFrame generates the job shell script using the following pattern: + +.. code-block:: bash + + #!/bin/bash -l + {job_scheduler_preamble} + {test_environment} + {prerun_cmds} + {parallel_launcher} {executable} {executable_opts} + {postrun_cmds} + +The ``job_scheduler_preamble`` contains the directives that control the job allocation. +The ``test_environment`` are the necessary commands for setting up the environment of the test. +This is the place where the modules and environment variables specified in :attr:`modules ` and :attr:`variables ` attributes are emitted. +Then the commands specified in :attr:`prerun_cmds ` follow, while those specified in the :attr:`postrun_cmds ` come after the launch of the parallel job. +The parallel launch itself consists of three parts: + +#. The parallel launcher program (e.g., ``srun``, ``mpirun`` etc.) with its options, +#. the regression test executable as specified in the :attr:`executable ` attribute and +#. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. + +A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. + + +Flexible Regression Tests +------------------------- + +.. versionadded:: 2.15 + +ReFrame can automatically set the number of tasks of a particular test, if its :attr:`num_tasks ` attribute is set to a negative value or zero. +In ReFrame's terminology, such tests are called *flexible*. +Negative values indicate the minimum number of tasks that are acceptable for this test (a value of ``-4`` indicates that at least ``4`` tasks are required). +A zero value indicates the default minimum number of tasks which is equal to :attr:`num_tasks_per_node `. + +By default, ReFrame will spawn such a test on all the idle nodes of the current system partition, but this behavior can be adjusted with the |--flex-alloc-nodes|_ command-line option. +Flexible tests are very useful for diagnostics tests, e.g., tests for checking the health of a whole set nodes. +In this example, we demonstrate this feature through a simple test that runs ``hostname``. +The test will verify that all the nodes print the expected host name: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + +The first thing to notice in this test is that :attr:`num_tasks ` is set to zero. +This is a requirement for flexible tests: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + :lines: 17 + :dedent: 8 + +The sanity function of this test simply counts the host names and verifies that they are as many as expected: + +.. literalinclude:: ../tutorial/advanced/advanced_example9.py + :lines: 19-22 + :dedent: 8 + +Notice, however, that the sanity check does not use :attr:`num_tasks` directly, but rather access the attribute through the :func:`sn.getattr() ` sanity function, which is a replacement for the :func:`getattr` builtin. +The reason for that is that at the time the sanity check expression is created, :attr:`num_tasks` is ``0`` and it will only be set to its actual value during the run phase. +Consequently, we need to defer the attribute retrieval, thus we use the :func:`sn.getattr() ` sanity function instead of accessing it directly + + +.. |--flex-alloc-nodes| replace:: :attr:`--flex-alloc-nodes` +.. _--flex-alloc-nodes: manpage.html#cmdoption-flex-alloc-nodes + + + +Testing containerized applications +---------------------------------- + +.. versionadded:: 2.20 + + +ReFrame can be used also to test applications that run inside a container. +A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. +The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + +A container-based test in ReFrame requires that the :attr:`container_platform ` is set: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 17 + +This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. +In this case, the test will be using `Singularity `__ as a container platform. +If such a platform is not configured for the current system, the test will fail. +For a complete list of supported container platforms, the user is referred to the `configuration reference `__. + +As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 17-20 + +These two attributes are mandatory for container-based check. +The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. +It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. + +In the above example, ReFrame will run the container as follows: + +.. code:: shell + + singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' + +By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. +The user commands then are then run from that directory one after the other. +Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. + +Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: + +.. literalinclude:: ../tutorial/advanced/advanced_example10.py + :lines: 21 + +Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: + +.. code-block:: python + + self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), + ('/path/to/host/dir2', '/path/to/container/mount_point2')] + + +For a complete list of the available attributes of a specific container platform, the reader is referred to `ReFrame Programming APIs `__ guide. diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 9e02aec23e..cd2ab696c4 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -5,6 +5,6 @@ ReFrame Tutorials .. toctree:: - tutorial_basic - tutorial_advanced + tutorial_basics + tutorial_misc_topics tutorial_deps diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index de88b87e73..6c9e761628 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -436,6 +436,9 @@ class RegressionTest(metaclass=RegressionTestMeta): int, type(None)) #: Number of GPUs per node required by this test. + #: This is attribute translated internally to the ``_rfm_gpu`` resource. + #: For more information on test resources, have a look at the + #: :attr:`extra_resources` attribute. #: #: :type: integral #: :default: ``0`` diff --git a/tutorial/advanced/advanced_example1.py b/tutorial/advanced/advanced_example1.py deleted file mode 100644 index 5f56d37e57..0000000000 --- a/tutorial/advanced/advanced_example1.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class MakefileTest(rfm.RegressionTest): - def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the use of Makefiles ' - 'and compile options') - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.executable = './advanced_example1' - self.build_system = 'Make' - self.build_system.cppflags = ['-DMESSAGE'] - self.sanity_patterns = sn.assert_found('SUCCESS', self.stdout) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} diff --git a/tutorial/advanced/advanced_example2.py b/tutorial/advanced/advanced_example2.py deleted file mode 100644 index 9f72488652..0000000000 --- a/tutorial/advanced/advanced_example2.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class ExampleRunOnlyTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the class' - 'RunOnlyRegressionTest') - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sourcesdir = None - - lower = 90 - upper = 100 - self.executable = 'echo "Random: $((RANDOM%({1}+1-{0})+{0}))"'.format( - lower, upper) - self.sanity_patterns = sn.assert_bounded(sn.extractsingle( - r'Random: (?P\S+)', self.stdout, 'number', float), - lower, upper) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} diff --git a/tutorial/advanced/src/Makefile b/tutorial/advanced/src/Makefile deleted file mode 100644 index 661016462f..0000000000 --- a/tutorial/advanced/src/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -EXECUTABLE := advanced_example1 - -.SUFFIXES: .o .c - -OBJS := advanced_example1.o - -$(EXECUTABLE): $(OBJS) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ - -$(OBJS): advanced_example1.c - $(CC) $(CPPFLAGS) $(CFLAGS) -c $(LDFLAGS) -o $@ $^ diff --git a/tutorial/advanced/src/Makefile_example4 b/tutorial/advanced/src/Makefile_example4 deleted file mode 100644 index 8a0a6456ed..0000000000 --- a/tutorial/advanced/src/Makefile_example4 +++ /dev/null @@ -1,16 +0,0 @@ -EXECUTABLE := advanced_example4 - -CPPFLAGS = -DCUDA_HOME=\"$(CUDA_HOME)\" - -.SUFFIXES: .o .c - -OBJS := advanced_example4.o - -$(EXECUTABLE): $(OBJS) - $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $^ - -$(OBJS): advanced_example4.c - $(CC) $(CPPFLAGS) $(CFLAGS) -c $(LDFLAGS) -o $@ $^ - -clean: - /bin/rm -f $(OBJS) $(EXECUTABLE) diff --git a/tutorial/example4.py b/tutorial/example4.py deleted file mode 100644 index 36f377e004..0000000000 --- a/tutorial/example4.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class Example4Test(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication example with OpenACC' - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_openacc.c' - self.build_system = 'SingleSource' - self.executable_opts = ['1024', '100'] - self.modules = ['craype-accel-nvidia60'] - self.num_gpus_per_node = 1 - self.prgenv_flags = { - 'PrgEnv-cray': ['-hacc', '-hnoomp'], - 'PrgEnv-pgi': ['-acc', '-ta=tesla:cc60'] - } - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - self.build_system.cflags = self.prgenv_flags[self.current_environ.name] diff --git a/tutorial/src/example_matrix_vector_multiplication_openacc.c b/tutorial/src/example_matrix_vector_multiplication_openacc.c deleted file mode 100644 index a3ed87fa80..0000000000 --- a/tutorial/src/example_matrix_vector_multiplication_openacc.c +++ /dev/null @@ -1,76 +0,0 @@ -#include -#include -#include - -int main (int argc, char *argv[]){ - double *matrix, *vector_in, *vector_out, out; - long dim_mn, iterations, i, j, iteration; - struct timeval start, stop, tdiff; - - if (argc!=3){ - fprintf(stderr, "%s matrixdimension numberofiterations\n", argv[0]); - exit(1); - } - dim_mn = atoi(argv[1]); - iterations = atoi(argv[2]); - - if ((dim_mn<1)||(iterations<1)){ - fprintf(stderr, "matrixdimension and numberofiterations must be " - "positive integers\n"); - exit(2); - } - - matrix = (double*) malloc(dim_mn*dim_mn*sizeof(double)); - vector_in = (double*) malloc(dim_mn*sizeof(double)); - vector_out = (double*) malloc(dim_mn*sizeof(double)); - for (i=0; i + + +int main() +{ + printf("Hello, World!\n"); + return 0; +} diff --git a/tutorials/basics/hello/src/hello.cpp b/tutorials/basics/hello/src/hello.cpp new file mode 100644 index 0000000000..e3d4e10984 --- /dev/null +++ b/tutorials/basics/hello/src/hello.cpp @@ -0,0 +1,13 @@ +// Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +// ReFrame Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include + + +int main() +{ + std::cout << "Hello, World!\n"; + return 0; +} diff --git a/tutorials/basics/hellomp/hellomp1.py b/tutorials/basics/hellomp/hellomp1.py new file mode 100644 index 0000000000..b6519efa33 --- /dev/null +++ b/tutorials/basics/hellomp/hellomp1.py @@ -0,0 +1,19 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class HelloThreadedTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sourcepath = 'hello_threads.cpp' + self.build_system = 'SingleSource' + self.build_system.cxxflags = ['-std=c++11', '-Wall'] + self.executable_opts = ['16'] + self.sanity_patterns = sn.assert_found(r'Hello, World\!', self.stdout) diff --git a/tutorials/basics/hellomp/hellomp2.py b/tutorials/basics/hellomp/hellomp2.py new file mode 100644 index 0000000000..aafa6fddb7 --- /dev/null +++ b/tutorials/basics/hellomp/hellomp2.py @@ -0,0 +1,21 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class HelloThreadedExtendedTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sourcepath = 'hello_threads.cpp' + self.executable_opts = ['16'] + self.build_system = 'SingleSource' + self.build_system.cxxflags = ['-std=c++11', '-Wall'] + num_messages = sn.len(sn.findall(r'\[\s?\d+\] Hello, World\!', + self.stdout)) + self.sanity_patterns = sn.assert_eq(num_messages, 16) diff --git a/tutorials/basics/hellomp/hellomp3.py b/tutorials/basics/hellomp/hellomp3.py new file mode 100644 index 0000000000..2a6a945dec --- /dev/null +++ b/tutorials/basics/hellomp/hellomp3.py @@ -0,0 +1,22 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class HelloThreadedExtended2Test(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + self.sourcepath = 'hello_threads.cpp' + self.executable_opts = ['16'] + self.build_system = 'SingleSource' + self.build_system.cppflags = ['-DSYNC_MESSAGES'] + self.build_system.cxxflags = ['-std=c++11', '-Wall'] + num_messages = sn.len(sn.findall(r'\[\s?\d+\] Hello, World\!', + self.stdout)) + self.sanity_patterns = sn.assert_eq(num_messages, 16) diff --git a/tutorials/basics/hellomp/src/hello_threads.cpp b/tutorials/basics/hellomp/src/hello_threads.cpp new file mode 100644 index 0000000000..b2c7a821fb --- /dev/null +++ b/tutorials/basics/hellomp/src/hello_threads.cpp @@ -0,0 +1,49 @@ +// Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +// ReFrame Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include +#include + + +#ifdef SYNC_MESSAGES +std::mutex hello_mutex; +#endif + + +void greetings(int tid) +{ +#ifdef SYNC_MESSAGES + const std::lock_guard lock(hello_mutex); +#endif + std::cout << "[" << std::setw(2) << tid << "] " << "Hello, World!\n"; +} + + +int main(int argc, char *argv[]) +{ + int nr_threads = 1; + if (argc > 1) { + nr_threads = std::atoi(argv[1]); + } + + if (nr_threads <= 0) { + std::cerr << "thread count must a be positive integer\n"; + return 1; + } + + std::vector threads; + for (auto i = 0; i < nr_threads; ++i) { + threads.push_back(std::thread(greetings, i)); + } + + for (auto &t : threads) { + t.join(); + } + + return 0; +} diff --git a/tutorials/basics/stream/stream1.py b/tutorials/basics/stream/stream1.py new file mode 100644 index 0000000000..d963eb2522 --- /dev/null +++ b/tutorials/basics/stream/stream1.py @@ -0,0 +1,37 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class StreamTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['gnu'] + self.prebuild_cmds = [ + 'wget http://www.cs.virginia.edu/stream/FTP/Code/stream.c', + ] + self.build_system = 'SingleSource' + self.sourcepath = 'stream.c' + self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] + self.build_system.cflags = ['-fopenmp', '-O3', '-Wall'] + self.variables = { + 'OMP_NUM_THREADS': '4', + 'OMP_PLACES': 'cores' + } + self.sanity_patterns = sn.assert_found(r'Solution Validates', + self.stdout) + self.perf_patterns = { + 'Copy': sn.extractsingle(r'Copy:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Scale': sn.extractsingle(r'Scale:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Add': sn.extractsingle(r'Add:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Triad': sn.extractsingle(r'Triad:\s+(\S+)\s+.*', + self.stdout, 1, float) + } diff --git a/tutorials/basics/stream/stream2.py b/tutorials/basics/stream/stream2.py new file mode 100644 index 0000000000..dbd094b206 --- /dev/null +++ b/tutorials/basics/stream/stream2.py @@ -0,0 +1,45 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class StreamWithRefTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['gnu'] + self.prebuild_cmds = [ + 'wget http://www.cs.virginia.edu/stream/FTP/Code/stream.c', + ] + self.build_system = 'SingleSource' + self.sourcepath = 'stream.c' + self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] + self.build_system.cflags = ['-fopenmp', '-O3', '-Wall'] + self.variables = { + 'OMP_NUM_THREADS': '4', + 'OMP_PLACES': 'cores' + } + self.sanity_patterns = sn.assert_found(r'Solution Validates', + self.stdout) + self.perf_patterns = { + 'Copy': sn.extractsingle(r'Copy:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Scale': sn.extractsingle(r'Scale:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Add': sn.extractsingle(r'Add:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Triad': sn.extractsingle(r'Triad:\s+(\S+)\s+.*', + self.stdout, 1, float) + } + self.reference = { + 'catalina': { + 'Copy': (25200, -0.05, 0.05, 'MB/s'), + 'Scale': (16800, -0.05, 0.05, 'MB/s'), + 'Add': (18500, -0.05, 0.05, 'MB/s'), + 'Triad': (18800, -0.05, 0.05, 'MB/s') + } + } diff --git a/tutorials/basics/stream/stream3.py b/tutorials/basics/stream/stream3.py new file mode 100644 index 0000000000..ba82b8bbe9 --- /dev/null +++ b/tutorials/basics/stream/stream3.py @@ -0,0 +1,70 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class StreamMultiSysTest(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['*'] + self.valid_prog_environs = ['cray', 'gnu', 'intel', 'pgi'] + self.prebuild_cmds = [ + 'wget http://www.cs.virginia.edu/stream/FTP/Code/stream.c', + ] + self.build_system = 'SingleSource' + self.sourcepath = 'stream.c' + self.build_system.cppflags = ['-DSTREAM_ARRAY_SIZE=$((1 << 25))'] + self.sanity_patterns = sn.assert_found(r'Solution Validates', + self.stdout) + self.perf_patterns = { + 'Copy': sn.extractsingle(r'Copy:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Scale': sn.extractsingle(r'Scale:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Add': sn.extractsingle(r'Add:\s+(\S+)\s+.*', + self.stdout, 1, float), + 'Triad': sn.extractsingle(r'Triad:\s+(\S+)\s+.*', + self.stdout, 1, float) + } + self.reference = { + 'catalina': { + 'Copy': (25200, -0.05, 0.05, 'MB/s'), + 'Scale': (16800, -0.05, 0.05, 'MB/s'), + 'Add': (18500, -0.05, 0.05, 'MB/s'), + 'Triad': (18800, -0.05, 0.05, 'MB/s') + } + } + + # Flags per programming environment + self.flags = { + 'cray': ['-fopenmp', '-O3', '-Wall'], + 'gnu': ['-fopenmp', '-O3', '-Wall'], + 'intel': ['-qopenmp', '-O3', '-Wall'], + 'pgi': ['-mp', '-O3'] + } + + # Number of cores for each system + self.cores = { + 'catalina:default': 4, + 'daint:gpu': 12, + 'daint:mc': 36, + 'daint:login': 10 + } + + @rfm.run_before('compile') + def setflags(self): + environ = self.current_environ.name + self.build_system.cflags = self.flags.get(environ, []) + + @rfm.run_before('run') + def set_num_threads(self): + num_threads = self.cores.get(self.current_partition.fullname, 1) + self.num_cpus_per_task = num_threads + self.variables = { + 'OMP_NUM_THREADS': str(num_threads), + 'OMP_PLACES': 'cores' + } diff --git a/tutorials/config/settings.py b/tutorials/config/settings.py new file mode 100644 index 0000000000..8e73eec0da --- /dev/null +++ b/tutorials/config/settings.py @@ -0,0 +1,163 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +# +# Tutorial settings +# + +site_configuration = { + 'systems': [ + { + 'name': 'catalina', + 'descr': 'My Mac', + 'hostnames': ['tresa'], + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'], + 'max_jobs': 10 + } + ] + }, + { + 'name': 'daint', + 'descr': 'Piz Daint Supercomputer', + 'hostnames': ['daint'], + 'modules_system': 'tmod32', + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'intel', 'pgi', 'cray'], + 'max_jobs': 10 + }, + { + 'name': 'gpu', + 'descr': 'Hybrid nodes', + 'scheduler': 'slurm', + 'launcher': 'srun', + 'access': ['-C gpu', '-A csstaff'], + 'environs': ['gnu', 'intel', 'pgi', 'cray'], + 'max_jobs': 100 + }, + { + 'name': 'mc', + 'descr': 'Multicore nodes', + 'scheduler': 'slurm', + 'launcher': 'srun', + 'access': ['-C mc', '-A csstaff'], + 'environs': ['gnu', 'intel', 'pgi', 'cray'], + 'max_jobs': 100 + } + ] + }, + { + 'name': 'generic', + 'descr': 'Generic example system', + 'hostnames': ['.*'], + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin'] + } + ] + }, + ], + 'environments': [ + { + 'name': 'gnu', + 'cc': 'gcc-9', + 'cxx': 'g++-9', + 'ftn': 'gfortran-9' + }, + { + 'name': 'gnu', + 'modules': ['PrgEnv-gnu'], + 'cc': 'cc', + 'cxx': 'CC', + 'ftn': 'ftn', + 'target_systems': ['daint'] + }, + { + 'name': 'cray', + 'modules': ['PrgEnv-cray'], + 'cc': 'cc', + 'cxx': 'CC', + 'ftn': 'ftn', + 'target_systems': ['daint'] + }, + { + 'name': 'intel', + 'modules': ['PrgEnv-intel'], + 'cc': 'cc', + 'cxx': 'CC', + 'ftn': 'ftn', + 'target_systems': ['daint'] + }, + { + 'name': 'pgi', + 'modules': ['PrgEnv-pgi'], + 'cc': 'cc', + 'cxx': 'CC', + 'ftn': 'ftn', + 'target_systems': ['daint'] + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'ftn': '' + }, + { + 'name': 'builtin', + 'cc': 'cc', + 'cxx': '', + 'ftn': '' + }, + ], + 'logging': [ + { + 'level': 'debug', + 'handlers': [ + { + 'type': 'stream', + 'name': 'stdout', + 'level': 'info', + 'format': '%(message)s' + }, + { + 'type': 'file', + 'name': 'reframe.log', + 'level': 'debug', + 'format': '[%(asctime)s] %(levelname)s: %(check_info)s: %(message)s', # noqa: E501 + 'append': False + } + ], + 'handlers_perflog': [ + { + 'type': 'filelog', + 'prefix': '%(check_system)s/%(check_partition)s', + 'level': 'info', + 'format': ( + '%(check_job_completion_time)s|reframe %(version)s|' + '%(check_info)s|jobid=%(check_jobid)s|' + '%(check_perf_var)s=%(check_perf_value)s|' + 'ref=%(check_perf_ref)s ' + '(l=%(check_perf_lower_thres)s, ' + 'u=%(check_perf_upper_thres)s)|' + '%(check_perf_unit)s' + ), + 'append': True + } + ] + } + ], +} diff --git a/tutorial/example5.py b/tutorials/misc/gpu/cuda.py similarity index 64% rename from tutorial/example5.py rename to tutorials/misc/gpu/cuda.py index e1a19aa9c7..15cd445a77 100644 --- a/tutorial/example5.py +++ b/tutorials/misc/gpu/cuda.py @@ -8,16 +8,14 @@ @rfm.simple_test -class Example5Test(rfm.RegressionTest): +class CUDATest(rfm.RegressionTest): def __init__(self): self.descr = 'Matrix-vector multiplication example with CUDA' self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_cuda.cu' + self.valid_prog_environs = ['cray', 'gnu', 'pgi'] + self.sourcepath = 'matvec.cu' self.executable_opts = ['1024', '100'] self.modules = ['cudatoolkit'] - self.num_gpus_per_node = 1 self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} + r'time for single matrix vector multiplication', self.stdout + ) diff --git a/tutorial/src/example_matrix_vector_multiplication_cuda.cu b/tutorials/misc/gpu/src/matvec.cu similarity index 100% rename from tutorial/src/example_matrix_vector_multiplication_cuda.cu rename to tutorials/misc/gpu/src/matvec.cu diff --git a/tutorials/misc/makefiles/maketest.py b/tutorials/misc/makefiles/maketest.py new file mode 100644 index 0000000000..bc4efe2a10 --- /dev/null +++ b/tutorials/misc/makefiles/maketest.py @@ -0,0 +1,33 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.parameterized_test(['float'], ['double']) +class MakefileTest(rfm.RegressionTest): + def __init__(self, elem_type): + self.descr = 'Test demonstrating use of Makefiles' + self.valid_systems = ['*'] + self.valid_prog_environs = ['clang', 'gnu'] + self.executable = './dotprod' + self.executable_opts = ['100000'] + self.build_system = 'Make' + self.build_system.cppflags = [f'-DELEM_TYPE={elem_type}'] + self.sanity_patterns = sn.assert_found( + rf'Result \({elem_type}\):', self.stdout + ) + + +@rfm.parameterized_test(['float'], ['double']) +class MakeOnlyTest(rfm.CompileOnlyRegressionTest): + def __init__(self, elem_type): + self.descr = 'Test demonstrating use of Makefiles' + self.valid_systems = ['*'] + self.valid_prog_environs = ['clang', 'gnu'] + self.build_system = 'Make' + self.build_system.cppflags = [f'-DELEM_TYPE={elem_type}'] + self.sanity_patterns = sn.assert_not_found(r'warning', self.stdout) diff --git a/tutorials/misc/makefiles/src/dotprod.cpp b/tutorials/misc/makefiles/src/dotprod.cpp new file mode 100644 index 0000000000..e0fdc715a0 --- /dev/null +++ b/tutorials/misc/makefiles/src/dotprod.cpp @@ -0,0 +1,72 @@ +// Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +// ReFrame Project Developers. See the top-level LICENSE file for details. +// +// SPDX-License-Identifier: BSD-3-Clause + +#include +#include +#include +#include + +#ifndef ELEM_TYPE +#define ELEM_TYPE double +#endif + +using elem_t = ELEM_TYPE; + +template +T dotprod(const std::vector &x, const std::vector &y) +{ + assert(x.size() == y.size()); + T sum = 0; + for (std::size_t i = 0; i < x.size(); ++i) { + sum += x[i] * y[i]; + } + + return sum; +} + +template +struct type_name { + static constexpr const char *value = nullptr; +}; + +template<> +struct type_name { + static constexpr const char *value = "float"; +}; + +template<> +struct type_name { + static constexpr const char *value = "double"; +}; + +int main(int argc, char *argv[]) +{ + if (argc < 2) { + std::cerr << argv[0] << ": too few arguments\n"; + std::cerr << "Usage: " << argv[0] << " DIM\n"; + return 1; + } + + std::size_t N = std::atoi(argv[1]); + if (N < 0) { + std::cerr << argv[0] + << ": array dimension must a positive integer: " << argv[1] + << "\n"; + return 1; + } + + std::vector x(N), y(N); + std::random_device seed; + std::mt19937 rand(seed()); + std::uniform_real_distribution<> dist(-1, 1); + for (std::size_t i = 0; i < N; ++i) { + x[i] = dist(rand); + y[i] = dist(rand); + } + + std::cout << "Result (" << type_name::value << "): " + << dotprod(x, y) << "\n"; + return 0; +} diff --git a/tutorial/advanced/advanced_example6.py b/tutorials/misc/random/randint.py similarity index 74% rename from tutorial/advanced/advanced_example6.py rename to tutorials/misc/random/randint.py index 9a481bef7b..6c585343f3 100644 --- a/tutorial/advanced/advanced_example6.py +++ b/tutorials/misc/random/randint.py @@ -10,15 +10,14 @@ @rfm.simple_test class DeferredIterationTest(rfm.RunOnlyRegressionTest): def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the use of deferred ' - 'iteration via the `map` sanity function.') + self.descr = 'Apply a sanity function iteratively' self.valid_systems = ['*'] self.valid_prog_environs = ['*'] self.executable = './random_numbers.sh' numbers = sn.extractall( - r'Random: (?P\S+)', self.stdout, 'number', float) + r'Random: (?P\S+)', self.stdout, 'number', float + ) self.sanity_patterns = sn.and_( sn.assert_eq(sn.count(numbers), 100), - sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers))) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} + sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers)) + ) diff --git a/tutorial/advanced/src/random_numbers.sh b/tutorials/misc/random/src/random_numbers.sh similarity index 50% rename from tutorial/advanced/src/random_numbers.sh rename to tutorials/misc/random/src/random_numbers.sh index 8a6e7e8d01..13e0336621 100755 --- a/tutorial/advanced/src/random_numbers.sh +++ b/tutorials/misc/random/src/random_numbers.sh @@ -1,4 +1,9 @@ #!/usr/bin/env bash +# +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause if [ -z $LOWER ]; then export LOWER=90 diff --git a/tutorials/misc/runonly/echorand.py b/tutorials/misc/runonly/echorand.py new file mode 100644 index 0000000000..20eaa220b5 --- /dev/null +++ b/tutorials/misc/runonly/echorand.py @@ -0,0 +1,28 @@ +# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class EchoRandTest(rfm.RunOnlyRegressionTest): + def __init__(self): + self.descr = 'A simple test that echoes a random number' + self.valid_systems = ['*'] + self.valid_prog_environs = ['*'] + lower = 90 + upper = 100 + self.executable = 'echo' + self.executable_opts = [ + 'Random: ', + f'$((RANDOM%({upper}+1-{lower})+{lower}))' + ] + self.sanity_patterns = sn.assert_bounded( + sn.extractsingle( + r'Random: (?P\S+)', self.stdout, 'number', float + ), + lower, upper + ) From f87ce8b416e64873a74cd97bc100bcb76c64fb16 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 4 Jul 2020 13:41:47 +0200 Subject: [PATCH 2/7] Renew the tutorial - The basic tutorial is rewritten. It starts from the vanilla installation of ReFrame and ends in porting the tests to a HPC cluster. - The ex-advanced tutorial is now renamed to "Miscellaneous topics" and it also hosts a couple of examples from former basic tutorial. - The structure and names of the tutorial examples is changed. No more generic names are used. - Other parts of the documentation were fine tuned and updated. --- ci-scripts/ci-runner.bash | 6 +- docs/configure.rst | 247 ++++++++---------- docs/dependencies.rst | 2 +- docs/regression_test_api.rst | 2 + docs/started.rst | 6 +- docs/tutorial_basics.rst | 8 +- docs/tutorial_deps.rst | 14 +- docs/tutorial_misc_topics.rst | 119 +++------ docs/tutorials.rst | 2 +- docs/usecases.rst | 2 +- tutorial/advanced/advanced_example4.py | 24 -- tutorial/advanced/advanced_example5.py | 23 -- tutorial/advanced/advanced_example8.py | 57 ---- tutorial/advanced/src/advanced_example1.c | 11 - tutorial/advanced/src/advanced_example4.c | 25 -- ..._matrix_vector_multiplication_mpi_openmp.c | 1 - ...mple_matrix_vector_multiplication_openmp.c | 1 - tutorial/advanced/src/scripts/limits.sh | 4 - tutorial/config/settings.py | 136 ---------- tutorial/example1.py | 20 -- tutorial/example2.py | 67 ----- tutorial/example3.py | 39 --- tutorial/example6.py | 32 --- tutorial/example7.py | 34 --- tutorial/example8.py | 111 -------- .../example_matrix_vector_multiplication.c | 67 ----- ..._matrix_vector_multiplication_mpi_openmp.c | 111 -------- ...mple_matrix_vector_multiplication_openmp.c | 72 ----- .../osu => tutorials/deps}/osu_benchmarks.py | 4 +- .../misc/containers/container_test.py | 6 +- .../misc/flexnodes/flextest.py | 5 +- tutorials/misc/makefiles/src/Makefile | 11 + .../misc/random/prepostrun.py | 16 +- .../misc/random}/src/limits.sh | 0 34 files changed, 195 insertions(+), 1090 deletions(-) delete mode 100644 tutorial/advanced/advanced_example4.py delete mode 100644 tutorial/advanced/advanced_example5.py delete mode 100644 tutorial/advanced/advanced_example8.py delete mode 100644 tutorial/advanced/src/advanced_example1.c delete mode 100644 tutorial/advanced/src/advanced_example4.c delete mode 120000 tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c delete mode 120000 tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c delete mode 100644 tutorial/advanced/src/scripts/limits.sh delete mode 100644 tutorial/config/settings.py delete mode 100644 tutorial/example1.py delete mode 100644 tutorial/example2.py delete mode 100644 tutorial/example3.py delete mode 100644 tutorial/example6.py delete mode 100644 tutorial/example7.py delete mode 100644 tutorial/example8.py delete mode 100644 tutorial/src/example_matrix_vector_multiplication.c delete mode 100644 tutorial/src/example_matrix_vector_multiplication_mpi_openmp.c delete mode 100644 tutorial/src/example_matrix_vector_multiplication_openmp.c rename {tutorial/advanced/osu => tutorials/deps}/osu_benchmarks.py (95%) rename tutorial/advanced/advanced_example10.py => tutorials/misc/containers/container_test.py (84%) rename tutorial/advanced/advanced_example9.py => tutorials/misc/flexnodes/flextest.py (78%) create mode 100644 tutorials/misc/makefiles/src/Makefile rename tutorial/advanced/advanced_example7.py => tutorials/misc/random/prepostrun.py (59%) rename {tutorial/advanced => tutorials/misc/random}/src/limits.sh (100%) diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 8bc2276974..847a7bf6de 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -41,8 +41,8 @@ checked_exec() run_tutorial_checks() { - cmd="./bin/reframe -C tutorial/config/settings.py \ ---save-log-files -r -t tutorial $@" + cmd="./bin/reframe -C tutorials/config/settings.py \ +--save-log-files -r -c tutorials/ -R -x HelloThreadedExtendedTest $@" echo "[INFO] Running tutorial checks with \`$cmd'" checked_exec $cmd } @@ -152,7 +152,7 @@ elif [ $CI_TUTORIAL -eq 1 ]; then # Run tutorial checks # Find modified or added tutorial checks tutorialchecks=( $(git diff origin/master...HEAD --name-only --oneline --no-merges | \ - grep -e '^tutorial/.*\.py') ) + grep -e '^tutorials/.*\.py') ) if [ ${#tutorialchecks[@]} -ne 0 ]; then tutorialchecks_path="" diff --git a/docs/configure.rst b/docs/configure.rst index 2dce364a82..d4d8d39664 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -5,8 +5,7 @@ Configuring ReFrame for Your Site ReFrame comes pre-configured with a minimal generic configuration that will allow you to run ReFrame on any system. This will allow you to run simple local tests using the default compiler of the system. Of course, ReFrame is much more powerful than that. -This section will guide you through configuring ReFrame for your HPC cluster. -We will use as a starting point a simplified configuration for the `Piz Daint `__ supercomputer at CSCS and we will elaborate along the way. +This section will guide you through configuring ReFrame for your site. If you started using ReFrame from version 3.0, you can keep on reading this section, otherwise you are advised to have a look first at the :doc:`migration_2_to_3` page. @@ -50,10 +49,10 @@ We'll refer to these top-level properties as *sections*. These sections contain other objects which further define in detail the framework's behavior. If you are using a Python file to configure ReFrame, this big JSON configuration object is stored in a special variable called ``site_configuration``. -We will explore the basic configuration of ReFrame through the following configuration file that permits ReFrame to run on Piz Daint. +We will explore the basic configuration of ReFrame by looking into the configuration file of the tutorials, which permits ReFrame to run both on the Piz Daint supercomputer and a local computer. For the complete listing and description of all configuration options, you should refer to the :doc:`config_reference`. -.. literalinclude:: ../tutorial/config/settings.py +.. literalinclude:: ../tutorials/config/settings.py :lines: 10- There are three required sections that each configuration file must provide: ``systems``, ``environments`` and ``logging``. @@ -68,8 +67,8 @@ ReFrame allows you to configure multiple systems in the same configuration file. Each system is a different object inside the ``systems`` section. In our example we define only one system, namely Piz Daint: -.. literalinclude:: ../tutorial/config/settings.py - :lines: 11-75 +.. literalinclude:: ../tutorials/config/settings.py + :lines: 11-73 Each system is associated with a set of properties, which in this case are the following: @@ -89,8 +88,8 @@ In the example shown here, we define three partitions that none of them correspo The ``login`` partition refers to the login nodes of the system, whereas the ``gpu`` and ``mc`` partitions refer to two different set of nodes in the same cluster that are effectively separated using Slurm constraints. Let's pick the ``gpu`` partition and look into it in more detail: -.. literalinclude:: ../tutorial/config/settings.py - :lines: 31-51 +.. literalinclude:: ../tutorials/config/settings.py + :lines: 31-58 The basic properties of a partition are the following: @@ -128,31 +127,18 @@ Environments in ReFrame are configured under the ``environments`` section of the In our configuration example for Piz Daint, we define each ReFrame environment to correspond to each of the Cray-provided programming environments. In other systems, you could define a ReFrame environment to wrap a toolchain (MPI + compiler combination): -.. literalinclude:: ../tutorial/config/settings.py - :lines: 76-93 +.. literalinclude:: ../tutorials/config/settings.py + :lines: 74-125 Each environment is associated with a name. This name will be used to reference this environment in different contexts, as for example in the ``environs`` property of the system partitions. -This environment definition is minimal, since the default values for the rest of the properties serve our purpose. +A programming environment in ReFrame is essentially a collection of environment modules, environment variables and compiler definitions. -An important feature in ReFrame's configuration, is that you can define section objects differently for different systems or system partitions. -In the following, for demonstration purposes, we define ``PrgEnv-gnu`` differently for the ``mc`` partition of the ``daint`` system (notice the condensed form of writing this as ``daint:mc``): - -.. code-block:: python - - { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu', 'openmpi'], - 'cc': 'mpicc', - 'cxx': 'mpicxx', - 'ftn': 'mpif90', - 'target_systems': ['daint:mc'] - } - -This environment loads different modules and sets the compilers differently, but the most important part is the ``target_systems`` property. -This property is a list of systems or system/partition combinations (as in this case) where this definition of the environment is in effect. -This means that ``PrgEnv-gnu`` will defined this way only for regression tests running on ``daint:mc``. -For all the other systems, it will be defined as shown before. +An important feature in ReFrame's configuration, is that you can define section objects differently for different systems or system partitions by using the ``target_systems`` property. +Notice, for example, how the ``gnu`` environment is defined differently for the system ``daint`` compared to the generic definition. +The ``target_systems`` property is a list of systems or system/partition combinations where this definition of the environment is in effect. +This means that ``gnu`` will defined this way only for regression tests running on ``daint``. +For all the other systems, it will be defined using the first definition. --------------------- @@ -163,8 +149,8 @@ ReFrame has a powerful logging mechanism that gives fine grained control over wh Additionally, it allows for logging performance data from performance tests into different channels. Let's see how logging is defined in our example configuration, which also represents a typical one for logging: -.. literalinclude:: ../tutorial/config/settings.py - :lines: 94-130 +.. literalinclude:: ../tutorials/config/settings.py + :lines: 126-162 Logging is configured under the ``logging`` section of the configuration, which is a list of logger objects. Unless you want to configure logging differently for different systems, a single logger object is enough. @@ -250,16 +236,16 @@ To better understand this, let's assume that we have the following ``environment 'environments': [ { - 'name': 'PrgEnv-cray', - 'modules': ['PrgEnv-cray'] + 'name': 'cray', + 'modules': ['cray'] }, { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu'] + 'name': 'gnu', + 'modules': ['gnu'] }, { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu', 'openmpi'], + 'name': 'gnu', + 'modules': ['gnu', 'openmpi'], 'cc': 'mpicc', 'cxx': 'mpicxx', 'ftn': 'mpif90', @@ -268,7 +254,7 @@ To better understand this, let's assume that we have the following ``environment ], -If the selected system is ``foo``, then ReFrame will use the second definition of ``PrgEnv-gnu`` which is specific to the ``foo`` system. +If the selected system is ``foo``, then ReFrame will use the second definition of ``gnu`` which is specific to the ``foo`` system. You can override completely the system auto-selection process by specifying a system or system/partition combination with the ``--system`` option, e.g., ``--system=daint`` or ``--system=daint:gpu``. @@ -284,135 +270,122 @@ Let's see some concrete examples: * Query the current system's partitions: - .. code:: + .. code-block:: console - ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=systems/0/partitions + ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=systems/0/partitions .. code:: javascript - [ - { - "name": "login", - "descr": "Login nodes", - "scheduler": "local", - "launcher": "local", - "environs": [ - "PrgEnv-cray", - "PrgEnv-gnu", - "PrgEnv-intel", - "PrgEnv-pgi" - ], - "max_jobs": 4 - }, - { - "name": "gpu", - "descr": "Hybrid nodes (Haswell/P100)", - "scheduler": "slurm", - "launcher": "srun", - "modules": [ - "daint-gpu" - ], - "access": [ - "--constraint=gpu" - ], - "environs": [ - "PrgEnv-cray", - "PrgEnv-gnu", - "PrgEnv-intel", - "PrgEnv-pgi" - ], - "container_platforms": [ - { - "name": "Singularity", - "modules": [ - "Singularity" - ] - } - ], - "max_jobs": 100 - }, - { - "name": "mc", - "descr": "Multicore nodes (Broadwell)", - "scheduler": "slurm", - "launcher": "srun", - "modules": [ - "daint-mc" - ], - "access": [ - "--constraint=mc" - ], - "environs": [ - "PrgEnv-cray", - "PrgEnv-gnu", - "PrgEnv-intel", - "PrgEnv-pgi" - ], - "container_platforms": [ - { - "name": "Singularity", - "modules": [ - "Singularity" - ] - } - ], - "max_jobs": 100 - } - ] + [ + { + "name": "login", + "descr": "Login nodes", + "scheduler": "local", + "launcher": "local", + "environs": [ + "gnu", + "intel", + "pgi", + "cray" + ], + "max_jobs": 10 + }, + { + "name": "gpu", + "descr": "Hybrid nodes", + "scheduler": "slurm", + "launcher": "srun", + "access": [ + "-C gpu", + "-A csstaff" + ], + "environs": [ + "gnu", + "intel", + "pgi", + "cray" + ], + "max_jobs": 100 + }, + { + "name": "mc", + "descr": "Multicore nodes", + "scheduler": "slurm", + "launcher": "srun", + "access": [ + "-C mc", + "-A csstaff" + ], + "environs": [ + "gnu", + "intel", + "pgi", + "cray" + ], + "max_jobs": 100 + } + ] Check how the output changes if we explicitly set system to ``daint:login``: - .. code:: + .. code-block:: console - ./bin/reframe -C tutorial/config/settings.py --system=daint:login --show-config=systems/0/partitions + ./bin/reframe -C tutorials/config/settings.py --system=daint:login --show-config=systems/0/partitions .. code:: javascript - [ - { - "name": "login", - "descr": "Login nodes", - "scheduler": "local", - "launcher": "local", - "environs": [ - "PrgEnv-cray", - "PrgEnv-gnu", - "PrgEnv-intel", - "PrgEnv-pgi" - ], - "max_jobs": 4 - } - ] + [ + { + "name": "login", + "descr": "Login nodes", + "scheduler": "local", + "launcher": "local", + "environs": [ + "gnu", + "intel", + "pgi", + "cray" + ], + "max_jobs": 10 + } + ] + ReFrame will internally represent system ``daint`` as having a single partition only. Notice also how you can use indexes to objects elements inside a list. * Query an environment configuration: - .. code:: + .. code-block:: console - ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=environments/@PrgEnv-gnu + ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=environments/@gnu .. code:: javascript - { - "name": "PrgEnv-gnu", - "modules": [ - "PrgEnv-gnu" - ] - } + { + "name": "gnu", + "modules": [ + "PrgEnv-gnu" + ], + "cc": "cc", + "cxx": "CC", + "ftn": "ftn", + "target_systems": [ + "daint" + ] + } If an object has a ``name`` property you can address it by name using the ``@name`` syntax, instead of its index. * Query an environment's compiler: - .. code:: + .. code-block:: console - ./bin/reframe -C tutorial/config/settings.py --system=daint --show-config=environments/@PrgEnv-gnu/cxx + ./bin/reframe -C tutorials/config/settings.py --system=daint --show-config=environments/@gnu/cxx .. code:: javascript "CC" - Notice that although the C++ compiler is not defined in the environment's definitions, ReFrame will print the default value, if you explicitly query its value. + If you explicitly query a configuration value and this value is not defined in the configuration file, ReFrame will print its default value. diff --git a/docs/dependencies.rst b/docs/dependencies.rst index 3f9073fdb3..1c79a26133 100644 --- a/docs/dependencies.rst +++ b/docs/dependencies.rst @@ -142,7 +142,7 @@ Resolving dependencies As shown in the :doc:`tutorial_deps`, test dependencies would be of limited usage if you were not able to use the results or information of the target tests. Let's reiterate over the :func:`set_executable` function of the :class:`OSULatencyTest` that we presented previously: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 32-38 The ``@require_deps`` decorator does some magic -- we will unravel this shortly -- with the function arguments of the :func:`set_executable` function and binds them to the target test dependencies by their name. diff --git a/docs/regression_test_api.rst b/docs/regression_test_api.rst index 5ca6b11bc5..aa17ea5f8f 100644 --- a/docs/regression_test_api.rst +++ b/docs/regression_test_api.rst @@ -120,6 +120,8 @@ It is up to the concrete build system implementations on how to use or not these :show-inheritance: +.. _container-platforms: + Container Platforms ------------------- diff --git a/docs/started.rst b/docs/started.rst index d57075fc1a..791230b90c 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -184,6 +184,6 @@ As soon as you configure ReFrame specifically for your system, you may rerun the Where to Go from Here --------------------- -The :doc:`configure` page guides you through the basic configuration aspects of ReFrame. -The :doc:`tutorials` will allow you to get a first idea on how to write and run ReFrame tests. -:doc:`topics` explain different aspects of the framework whereas the :doc:`manuals` provide complete reference guides for the command line interface, the configuration parameters and the programming APIs for writing tests. +The easiest way to start with ReFrame is to go through :doc:`tutorial_basics`, which will guide you step-by-step in both writing your first tests and in configuring ReFrame. +The :doc:`configure` page provides more details on the basic configuration aspects of ReFrame. +:doc:`topics` explain different aspects of the framework whereas :doc:`manuals` provide complete reference guides for the command line interface, the configuration parameters and the programming APIs for writing tests. diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index 2906b76d7b..44b4ceb7a5 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -4,14 +4,10 @@ .. versionadded:: 3.1 -.. |tutorialdir| replace:: :obj:`tutorials/` -.. |tutorialdir_basics| replace:: :obj:`tutorials/basics` -.. _tutorialdir_basics: https://github.com/eth-cscs/reframe/tree/master/tutorials/basics - This tutorial will give you a first overview of ReFrame and will acquaint with its basic concepts. We will start with a simple "Hello, World!" test running with the default configuration and we will expand the example along the way. We will also explore performance tests and we will port our tests to an HPC cluster. -The examples of this tutorial can be found in |tutorialdir_basics|_. +The examples of this tutorial can be found under :obj:`tutorials/basics/`. Getting Ready @@ -38,6 +34,7 @@ As simple as it may sound, a series of "naive" "Hello, World!" tests can reveal Here is its C version: .. literalinclude:: ../tutorials/basics/hello/src/hello.c + :language: c :lines: 6- @@ -321,6 +318,7 @@ We extend our C++ "Hello, World!" example to print the greetings from multiple t .. literalinclude:: ../tutorials/basics/hellomp/src/hello_threads.cpp + :language: cpp :lines: 6- This program takes as argument the number of threads it will create and it uses ``std::thread``, which is C++11 addition, meaning that we will need to pass ``-std=c++11`` to our compilers. diff --git a/docs/tutorial_deps.rst b/docs/tutorial_deps.rst index 0c22755a0c..e094b245a6 100644 --- a/docs/tutorial_deps.rst +++ b/docs/tutorial_deps.rst @@ -1,5 +1,5 @@ =============================================== -Tutorial 3: Using Dependencies in ReFrame Tests +Tutorial 2: Using Dependencies in ReFrame Tests =============================================== .. versionadded:: 2.21 @@ -12,7 +12,7 @@ You could have a build test, which all runtime tests would depend on. This is the approach we take with the following example, that fetches, builds and runs several `OSU benchmarks `__. We first create a basic compile-only test, that fetches the benchmarks and builds them for the different programming environments: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 92-106 There is nothing particular to that test, except perhaps that you can set :attr:`sourcesdir ` to ``None`` even for a test that needs to compile something. @@ -21,7 +21,7 @@ In such a case, you should at least provide the commands that fetch the code ins For the next test we need to use the OSU benchmark binaries that we just built, so as to run the MPI ping-pong benchmark. Here is the relevant part: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 12-44 First, since we will have multiple similar benchmarks, we move all the common functionality to the :class:`OSUBenchmarkTestBase` base class. @@ -29,7 +29,7 @@ Again nothing new here; we are going to use two nodes for the benchmark and we s The new part comes in with the :class:`OSULatencyTest` test in the following line: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 32 Here we tell ReFrame that this test depends on a test named ``OSUBuildTest``. @@ -45,7 +45,7 @@ If any of its dependencies fails, the current test will be marked as failure as The next step for the :class:`OSULatencyTest` is to set its executable to point to the binary produced by the :class:`OSUBuildTest`. This is achieved with the following specially decorated function: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 37-43 The :func:`@require_deps ` decorator will bind the arguments passed to the decorated function to the result of the dependency that each argument names. @@ -61,12 +61,12 @@ This concludes the presentation of the :class:`OSULatencyTest` test. The :class: The :class:`OSUAllreduceTest` shown below is similar to the other two, except that it is parameterized. It is essentially a scalability test that is running the ``osu_allreduce`` executable created by the :class:`OSUBuildTest` for 2, 4, 8 and 16 nodes. -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py :lines: 69-89 The full set of OSU example tests is shown below: -.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py +.. literalinclude:: ../tutorials/deps/osu_benchmarks.py Notice that the order in which dependencies are defined in a test file is irrelevant. In this case, we define :class:`OSUBuildTest` at the end. diff --git a/docs/tutorial_misc_topics.rst b/docs/tutorial_misc_topics.rst index 5c26f7eb9f..ae67b68412 100644 --- a/docs/tutorial_misc_topics.rst +++ b/docs/tutorial_misc_topics.rst @@ -1,14 +1,5 @@ -.. |tutorialdir_adv| replace:: :obj:`tutorial/advanced/` -.. _tutorialdir_adv: https://github.com/eth-cscs/reframe/tree/master/tutorial/advanced -.. |tutorial_adv_example1| replace:: :obj:`tutorial/advanced/src/advanced_example1.c` -.. _tutorial_adv_example1: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/advanced_example1.c -.. |limits.sh| replace:: :obj:`scripts/limits.sh` -.. _limits.sh: https://github.com/eth-cscs/reframe/blob/master/tutorial/advanced/src/scripts/limits.sh - - - ================================== - Tutorial 2: Miscellaneous topics + Tutorial 3: Miscellaneous topics ================================== This page collects several smaller tutorials that show specific parts of ReFrame. @@ -241,40 +232,37 @@ For more information about how exactly sanity functions work and how their execu ReFrame offers also the :func:`allx() ` sanity function which, conversely to the builtin :func:`all()` function, will return :class:`False` if its iterable argument is empty. -Customizing the Generated Job Script ------------------------------------- +Customizing the Test Job Script +------------------------------- -It is often the case that you must run some commands before or after the parallel launch of your executable. +It is often the case that we need to run some commands before or after the parallel launch of our executable. This can be easily achieved by using the :attr:`prerun_cmds ` and :attr:`postrun_cmds ` attributes of a ReFrame test. -The following example is a slightly modified version of the previous one. -The lower and upper limits for the random numbers are now set inside a helper shell script in |limits.sh|_ and we want also to print the word ``FINISHED`` after our executable has finished. -In order to achieve this, we need to source the helper script just before launching the executable and ``echo`` the desired message just after it finishes. -Here is the test file: +The following example is a slightly modified version of the random numbers test presented `above <#applying-a-sanity-function-iteratively>`__. +The lower and upper limits for the random numbers are now set inside a helper shell script in ``limits.sh`` located in the test's resources, which we need to source before running our tests. +Additionally, we want also to print ``FINISHED`` after our executable has finished. +Here is the modified test file: -.. literalinclude:: ../tutorial/advanced/advanced_example7.py +.. literalinclude:: ../tutorials/misc/random/prepostrun.py + :lines: 6- + :emphasize-lines: 11-12,17,20-21 + +The :attr:`prerun_cmds` and :attr:`postrun_cmds` are lists of commands to be emitted in the generated job script before and after the parallel launch of the executable. +Obviously, the working directory for these commands is that of the job script itself, which is the stage directory of the test. +The generated job script for this test looks like the following: + +.. code-block:: console -Notice the use of the :attr:`prerun_cmds` and :attr:`postrun_cmds` attributes. -These are lists of shell commands that are emitted verbatim in the job script. -The generated job script for this example is the following: + cat output/catalina/default/gnu/PrepostRunTest/rfm_PrepostRunTest_job.sh .. code-block:: bash - #!/bin/bash -l - #SBATCH --job-name="prerun_demo_check_daint_gpu_PrgEnv-gnu" - #SBATCH --time=0:10:0 - #SBATCH --ntasks=1 - #SBATCH --output=prerun_demo_check.out - #SBATCH --error=prerun_demo_check.err - #SBATCH --constraint=gpu - module load daint-gpu - module unload PrgEnv-cray - module load PrgEnv-gnu - source scripts/limits.sh - srun ./random_numbers.sh + #!/bin/bash + source limits.sh + ./random_numbers.sh echo FINISHED -ReFrame generates the job shell script using the following pattern: +Generally, ReFrame generates the job shell scripts using the following pattern: .. code-block:: bash @@ -285,9 +273,9 @@ ReFrame generates the job shell script using the following pattern: {parallel_launcher} {executable} {executable_opts} {postrun_cmds} -The ``job_scheduler_preamble`` contains the directives that control the job allocation. +The ``job_scheduler_preamble`` contains the backend job scheduler directives that control the job allocation. The ``test_environment`` are the necessary commands for setting up the environment of the test. -This is the place where the modules and environment variables specified in :attr:`modules ` and :attr:`variables ` attributes are emitted. +These include any modules or environment variables set at the `system partition level `__ or any `modules `__ or `environment variables `__ set at the test level. Then the commands specified in :attr:`prerun_cmds ` follow, while those specified in the :attr:`postrun_cmds ` come after the launch of the parallel job. The parallel launch itself consists of three parts: @@ -295,8 +283,6 @@ The parallel launch itself consists of three parts: #. the regression test executable as specified in the :attr:`executable ` attribute and #. the options to be passed to the executable as specified in the :attr:`executable_opts ` attribute. -A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it. - Flexible Regression Tests ------------------------- @@ -313,21 +299,13 @@ Flexible tests are very useful for diagnostics tests, e.g., tests for checking t In this example, we demonstrate this feature through a simple test that runs ``hostname``. The test will verify that all the nodes print the expected host name: -.. literalinclude:: ../tutorial/advanced/advanced_example9.py +.. literalinclude:: ../tutorials/misc/flexnodes/flextest.py + :lines: 6- + :emphasize-lines: 11-16 The first thing to notice in this test is that :attr:`num_tasks ` is set to zero. -This is a requirement for flexible tests: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 17 - :dedent: 8 - -The sanity function of this test simply counts the host names and verifies that they are as many as expected: - -.. literalinclude:: ../tutorial/advanced/advanced_example9.py - :lines: 19-22 - :dedent: 8 - +This is a requirement for flexible tests. +The sanity check of this test simply counts the host names printed and verifies that they are as many as expected. Notice, however, that the sanity check does not use :attr:`num_tasks` directly, but rather access the attribute through the :func:`sn.getattr() ` sanity function, which is a replacement for the :func:`getattr` builtin. The reason for that is that at the time the sanity check expression is created, :attr:`num_tasks` is ``0`` and it will only be set to its actual value during the run phase. Consequently, we need to defer the attribute retrieval, thus we use the :func:`sn.getattr() ` sanity function instead of accessing it directly @@ -345,45 +323,31 @@ Testing containerized applications ReFrame can be used also to test applications that run inside a container. -A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform `. -The following example shows a simple test that runs some basic commands inside an Ubuntu 18.04 container and checks that the test has indeed run inside the container and that the stage directory was correctly mounted: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py +Here is a test that does so: -A container-based test in ReFrame requires that the :attr:`container_platform ` is set: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 17 +.. literalinclude:: ../tutorials/misc/containers/container_test.py + :lines: 6- + :emphasize-lines: 11-16 -This attribute accepts a string that corresponds to the name of the platform and it instantiates the appropriate :class:`ContainerPlatform ` object behind the scenes. +A container-based test can be written as :class:`RunOnlyRegressionTest ` that sets the :attr:`container_platform ` attribute. +This attribute accepts a string that corresponds to the name of the container platform that will be used to run the container for this test. In this case, the test will be using `Singularity `__ as a container platform. -If such a platform is not configured for the current system, the test will fail. -For a complete list of supported container platforms, the user is referred to the `configuration reference `__. - -As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container: +If such a platform is not `configured `__ for the current system, the test will fail. -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 17-20 - -These two attributes are mandatory for container-based check. -The :attr:`image ` attribute specifies the name of an image from a registry, whereas the :attr:`commands ` attribute provides the list of commands to be run inside the container. +As soon as the container platform to be used is defined, you need to specify the container image to use and the commands to run inside the container by setting the :attr:`image ` and the :attr:`commands ` container platform attributes. +These two attributes are mandatory for container-based checks. It is important to note that the :attr:`executable ` and :attr:`executable_opts ` attributes of the actual test are ignored in case of container-based tests. -In the above example, ReFrame will run the container as follows: +ReFrame will run the container as follows: -.. code:: shell +.. code-block:: console singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. The user commands then are then run from that directory one after the other. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. - Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: - -.. literalinclude:: ../tutorial/advanced/advanced_example10.py - :lines: 21 - Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: .. code-block:: python @@ -391,5 +355,4 @@ Besides the stage directory, additional mount points can be specified through th self.container_platform.mount_points = [('/path/to/host/dir1', '/path/to/container/mount_point1'), ('/path/to/host/dir2', '/path/to/container/mount_point2')] - -For a complete list of the available attributes of a specific container platform, the reader is referred to `ReFrame Programming APIs `__ guide. +For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. diff --git a/docs/tutorials.rst b/docs/tutorials.rst index cd2ab696c4..186a900631 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -6,5 +6,5 @@ ReFrame Tutorials .. toctree:: tutorial_basics - tutorial_misc_topics tutorial_deps + tutorial_misc_topics diff --git a/docs/usecases.rst b/docs/usecases.rst index f73b951008..0434fc4e9e 100644 --- a/docs/usecases.rst +++ b/docs/usecases.rst @@ -80,7 +80,7 @@ For example, some performance tests are tagged as "daily", others as "weekly", " ReFrame has also been integrated into NERSC's centralized data collection service used for facility and system monitoring, called the "Data Collect." The Data Collect stores data in an Elasticsearch instance, uses `Logstash `__ to ingest log information about the Cori system, and provides a web-based GUI to display results via `Kibana `__. -Cray, in turn, provides the `Cray Lightweight Log Manager `__ on XC systems such as Cori, which provides a syslog interface. +Cray, in turn, provides the Cray Lightweight Log Manager on XC systems such as Cori, which provides a syslog interface. ReFrame's support for Syslog, and the Python standard `logging `__ library, enabled simple integration with NERSC's Data Collect The result of this integration with ReFrame to the Data Collect is that the results from each ReFrame test executed on Cori are visible via a Kibana query within a few seconds of the test completing. One can then configure Elasticsearch to alert a system administrator if a particular system functionality stops working, or if the performance of certain benchmarks suddenly declines. diff --git a/tutorial/advanced/advanced_example4.py b/tutorial/advanced/advanced_example4.py deleted file mode 100644 index 6faea8657c..0000000000 --- a/tutorial/advanced/advanced_example4.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class EnvironmentVariableTest(rfm.RegressionTest): - def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the use' - 'of environment variables provided by loaded modules') - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['*'] - self.modules = ['cudatoolkit'] - self.variables = {'CUDA_HOME': '$CUDATOOLKIT_HOME'} - self.executable = './advanced_example4' - self.build_system = 'Make' - self.build_system.makefile = 'Makefile_example4' - self.sanity_patterns = sn.assert_found(r'SUCCESS', self.stdout) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} diff --git a/tutorial/advanced/advanced_example5.py b/tutorial/advanced/advanced_example5.py deleted file mode 100644 index 9dcc1585fb..0000000000 --- a/tutorial/advanced/advanced_example5.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class TimeLimitTest(rfm.RunOnlyRegressionTest): - def __init__(self): - self.descr = ('ReFrame tutorial demonstrating the use ' - 'of a user-defined time limit') - self.valid_systems = ['daint:gpu', 'daint:mc'] - self.valid_prog_environs = ['*'] - self.time_limit = '1m' - self.executable = 'sleep' - self.executable_opts = ['100'] - self.sanity_patterns = sn.assert_found( - r'CANCELLED.*DUE TO TIME LIMIT', self.stderr) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} diff --git a/tutorial/advanced/advanced_example8.py b/tutorial/advanced/advanced_example8.py deleted file mode 100644 index f01b1c519f..0000000000 --- a/tutorial/advanced/advanced_example8.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.parameterized_test(['MPI'], ['OpenMP']) -class MatrixVectorTest(rfm.RegressionTest): - def __init__(self, variant): - self.descr = 'Matrix-vector multiplication test (%s)' % variant - self.valid_systems = ['daint:gpu', 'daint:mc'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.build_system = 'SingleSource' - self.prgenv_flags = { - 'PrgEnv-cray': ['-homp'], - 'PrgEnv-gnu': ['-fopenmp'], - 'PrgEnv-intel': ['-openmp'], - 'PrgEnv-pgi': ['-mp'] - } - - if variant == 'MPI': - self.num_tasks = 8 - self.num_tasks_per_node = 2 - self.num_cpus_per_task = 4 - self.sourcepath = 'example_matrix_vector_multiplication_mpi_openmp.c' - elif variant == 'OpenMP': - self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' - self.num_cpus_per_task = 4 - - self.variables = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task) - } - matrix_dim = 1024 - iterations = 100 - self.executable_opts = [str(matrix_dim), str(iterations)] - - expected_norm = matrix_dim - found_norm = sn.extractsingle( - r'The L2 norm of the resulting vector is:\s+(?P\S+)', - self.stdout, 'norm', float) - self.sanity_patterns = sn.all([ - sn.assert_found( - r'time for single matrix vector multiplication', self.stdout), - sn.assert_lt(sn.abs(expected_norm - found_norm), 1.0e-6) - ]) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - if self.prgenv_flags is not None: - env = self.current_environ.name - self.build_system.cflags = self.prgenv_flags[env] diff --git a/tutorial/advanced/src/advanced_example1.c b/tutorial/advanced/src/advanced_example1.c deleted file mode 100644 index cae20d349c..0000000000 --- a/tutorial/advanced/src/advanced_example1.c +++ /dev/null @@ -1,11 +0,0 @@ -#include - -int main(){ -#ifdef MESSAGE - char *message = "SUCCESS"; -#else - char *message = "FAILURE"; -#endif - printf("Setting of preprocessor variable: %s\n", message); - return 0; -} diff --git a/tutorial/advanced/src/advanced_example4.c b/tutorial/advanced/src/advanced_example4.c deleted file mode 100644 index 9b465774ff..0000000000 --- a/tutorial/advanced/src/advanced_example4.c +++ /dev/null @@ -1,25 +0,0 @@ -#include -#include -#include - -#ifndef CUDA_HOME -# define CUDA_HOME "" -#endif - -int main() { - char *cuda_home_compile = CUDA_HOME; - char *cuda_home_runtime = getenv("CUDA_HOME"); - if (cuda_home_runtime && - strnlen(cuda_home_runtime, 256) && - strnlen(cuda_home_compile, 256) && - !strncmp(cuda_home_compile, cuda_home_runtime, 256)) { - printf("SUCCESS\n"); - } else { - printf("FAILURE\n"); - printf("Compiled with CUDA_HOME=%s, ran with CUDA_HOME=%s\n", - cuda_home_compile, - cuda_home_runtime ? cuda_home_runtime : ""); - } - - return 0; -} diff --git a/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c b/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c deleted file mode 120000 index 0c2c685c39..0000000000 --- a/tutorial/advanced/src/example_matrix_vector_multiplication_mpi_openmp.c +++ /dev/null @@ -1 +0,0 @@ -../../src/example_matrix_vector_multiplication_mpi_openmp.c \ No newline at end of file diff --git a/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c b/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c deleted file mode 120000 index bfee770f9f..0000000000 --- a/tutorial/advanced/src/example_matrix_vector_multiplication_openmp.c +++ /dev/null @@ -1 +0,0 @@ -../../src/example_matrix_vector_multiplication_openmp.c \ No newline at end of file diff --git a/tutorial/advanced/src/scripts/limits.sh b/tutorial/advanced/src/scripts/limits.sh deleted file mode 100644 index 67983a99eb..0000000000 --- a/tutorial/advanced/src/scripts/limits.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -export LOWER=50 -export UPPER=80 diff --git a/tutorial/config/settings.py b/tutorial/config/settings.py deleted file mode 100644 index 83097d1e2e..0000000000 --- a/tutorial/config/settings.py +++ /dev/null @@ -1,136 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -# -# Minimal settings for ReFrame's tutorial for running on Piz Daint -# - -site_configuration = { - 'systems': [ - { - 'name': 'daint', - 'descr': 'Piz Daint', - 'hostnames': ['daint'], - 'modules_system': 'tmod', - 'partitions': [ - { - 'name': 'login', - 'descr': 'Login nodes', - 'scheduler': 'local', - 'launcher': 'local', - 'environs': [ - 'PrgEnv-cray', - 'PrgEnv-gnu', - 'PrgEnv-intel', - 'PrgEnv-pgi' - ], - 'max_jobs': 4, - }, - { - 'name': 'gpu', - 'descr': 'Hybrid nodes (Haswell/P100)', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'modules': ['daint-gpu'], - 'access': ['--constraint=gpu'], - 'environs': [ - 'PrgEnv-cray', - 'PrgEnv-gnu', - 'PrgEnv-intel', - 'PrgEnv-pgi' - ], - 'container_platforms': [ - { - 'type': 'Singularity', - 'modules': ['Singularity'] - } - ], - 'max_jobs': 100, - }, - { - 'name': 'mc', - 'descr': 'Multicore nodes (Broadwell)', - 'scheduler': 'slurm', - 'launcher': 'srun', - 'modules': ['daint-mc'], - 'access': ['--constraint=mc'], - 'environs': [ - 'PrgEnv-cray', - 'PrgEnv-gnu', - 'PrgEnv-intel', - 'PrgEnv-pgi' - ], - 'container_platforms': [ - { - 'type': 'Singularity', - 'modules': ['Singularity'] - } - ], - 'max_jobs': 100, - } - ] - } - ], - 'environments': [ - { - 'name': 'PrgEnv-cray', - 'modules': ['PrgEnv-cray'] - }, - { - 'name': 'PrgEnv-gnu', - 'modules': ['PrgEnv-gnu'] - }, - { - 'name': 'PrgEnv-intel', - 'modules': ['PrgEnv-intel'] - }, - { - 'name': 'PrgEnv-pgi', - 'modules': ['PrgEnv-pgi'] - } - ], - 'logging': [ - { - 'level': 'debug', - 'handlers': [ - { - 'type': 'file', - 'name': 'reframe.log', - 'level': 'debug', - 'format': '[%(asctime)s] %(levelname)s: %(check_name)s: %(message)s', # noqa: E501 - 'append': False - }, - { - 'type': 'stream', - 'name': 'stdout', - 'level': 'info', - 'format': '%(message)s' - }, - { - 'type': 'file', - 'name': 'reframe.out', - 'level': 'info', - 'format': '%(message)s', - 'append': False - } - ], - 'handlers_perflog': [ - { - 'type': 'filelog', - 'prefix': '%(check_system)s/%(check_partition)s', - 'level': 'info', - 'format': '%(check_job_completion_time)s|reframe %(version)s|%(check_info)s|jobid=%(check_jobid)s|%(check_perf_var)s=%(check_perf_value)s|ref=%(check_perf_ref)s (l=%(check_perf_lower_thres)s, u=%(check_perf_upper_thres)s)', # noqa: E501 - 'datefmt': '%FT%T%:z', - 'append': True - } - ] - } - ], - 'general': [ - { - 'check_search_path': ['tutorial/'], - } - ] -} diff --git a/tutorial/example1.py b/tutorial/example1.py deleted file mode 100644 index 56da3bd0f0..0000000000 --- a/tutorial/example1.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - -@rfm.simple_test -class Example1Test(rfm.RegressionTest): - def __init__(self): - self.descr = 'Simple matrix-vector multiplication example' - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sourcepath = 'example_matrix_vector_multiplication.c' - self.executable_opts = ['1024', '100'] - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} diff --git a/tutorial/example2.py b/tutorial/example2.py deleted file mode 100644 index 14890eff91..0000000000 --- a/tutorial/example2.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class Example2aTest(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication example with OpenMP' - self.valid_systems = ['*'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' - self.build_system = 'SingleSource' - self.executable_opts = ['1024', '100'] - self.variables = { - 'OMP_NUM_THREADS': '4' - } - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - env = self.current_environ.name - if env == 'PrgEnv-cray': - self.build_system.cflags = ['-homp'] - elif env == 'PrgEnv-gnu': - self.build_system.cflags = ['-fopenmp'] - elif env == 'PrgEnv-intel': - self.build_system.cflags = ['-openmp'] - elif env == 'PrgEnv-pgi': - self.build_system.cflags = ['-mp'] - - -@rfm.simple_test -class Example2bTest(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication example with OpenMP' - self.valid_systems = ['*'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' - self.build_system = 'SingleSource' - self.executable_opts = ['1024', '100'] - self.prgenv_flags = { - 'PrgEnv-cray': ['-homp'], - 'PrgEnv-gnu': ['-fopenmp'], - 'PrgEnv-intel': ['-openmp'], - 'PrgEnv-pgi': ['-mp'] - } - self.variables = { - 'OMP_NUM_THREADS': '4' - } - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - self.build_system.cflags = self.prgenv_flags[self.current_environ.name] diff --git a/tutorial/example3.py b/tutorial/example3.py deleted file mode 100644 index d10eef6923..0000000000 --- a/tutorial/example3.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class Example3Test(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication example with MPI' - self.valid_systems = ['daint:gpu', 'daint:mc'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_mpi_openmp.c' - self.executable_opts = ['1024', '10'] - self.build_system = 'SingleSource' - self.prgenv_flags = { - 'PrgEnv-cray': ['-homp'], - 'PrgEnv-gnu': ['-fopenmp'], - 'PrgEnv-intel': ['-openmp'], - 'PrgEnv-pgi': ['-mp'] - } - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.num_tasks = 8 - self.num_tasks_per_node = 2 - self.num_cpus_per_task = 4 - self.variables = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task) - } - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - self.build_system.cflags = self.prgenv_flags[self.current_environ.name] diff --git a/tutorial/example6.py b/tutorial/example6.py deleted file mode 100644 index 079e40299d..0000000000 --- a/tutorial/example6.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class Example6Test(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication with L2 norm check' - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.sourcepath = 'example_matrix_vector_multiplication.c' - - matrix_dim = 1024 - iterations = 100 - self.executable_opts = [str(matrix_dim), str(iterations)] - - expected_norm = matrix_dim - found_norm = sn.extractsingle( - r'The L2 norm of the resulting vector is:\s+(?P\S+)', - self.stdout, 'norm', float) - self.sanity_patterns = sn.all([ - sn.assert_found( - r'time for single matrix vector multiplication', self.stdout), - sn.assert_lt(sn.abs(expected_norm - found_norm), 1.0e-6) - ]) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} diff --git a/tutorial/example7.py b/tutorial/example7.py deleted file mode 100644 index 4e51454585..0000000000 --- a/tutorial/example7.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -@rfm.simple_test -class Example7Test(rfm.RegressionTest): - def __init__(self): - self.descr = 'Matrix-vector multiplication (CUDA performance test)' - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_cuda.cu' - self.build_system = 'SingleSource' - self.build_system.cxxflags = ['-O3'] - self.executable_opts = ['4096', '1000'] - self.modules = ['cudatoolkit'] - self.num_gpus_per_node = 1 - self.sanity_patterns = sn.assert_found( - r'time for single matrix vector multiplication', self.stdout) - self.perf_patterns = { - 'perf': sn.extractsingle(r'Performance:\s+(?P\S+) Gflop/s', - self.stdout, 'Gflops', float) - } - self.reference = { - 'daint:gpu': { - 'perf': (50.0, -0.1, 0.1, 'Gflop/s'), - } - } - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} diff --git a/tutorial/example8.py b/tutorial/example8.py deleted file mode 100644 index 7b04acab2f..0000000000 --- a/tutorial/example8.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright 2016-2020 Swiss National Supercomputing Centre (CSCS/ETH Zurich) -# ReFrame Project Developers. See the top-level LICENSE file for details. -# -# SPDX-License-Identifier: BSD-3-Clause - -import reframe as rfm -import reframe.utility.sanity as sn - - -class BaseMatrixVectorTest(rfm.RegressionTest): - def __init__(self, test_version): - self.descr = '%s matrix-vector multiplication' % test_version - self.valid_systems = ['*'] - self.valid_prog_environs = ['*'] - self.build_system = 'SingleSource' - self.prgenv_flags = None - - matrix_dim = 1024 - iterations = 100 - self.executable_opts = [str(matrix_dim), str(iterations)] - - expected_norm = matrix_dim - found_norm = sn.extractsingle( - r'The L2 norm of the resulting vector is:\s+(?P\S+)', - self.stdout, 'norm', float) - self.sanity_patterns = sn.all([ - sn.assert_found( - r'time for single matrix vector multiplication', self.stdout), - sn.assert_lt(sn.abs(expected_norm - found_norm), 1.0e-6) - ]) - self.maintainers = ['you-can-type-your-email-here'] - self.tags = {'tutorial'} - - @rfm.run_before('compile') - def setflags(self): - if self.prgenv_flags is not None: - env = self.current_environ.name - self.build_system.cflags = self.prgenv_flags[env] - - -@rfm.simple_test -class SerialTest(BaseMatrixVectorTest): - def __init__(self): - super().__init__('Serial') - self.sourcepath = 'example_matrix_vector_multiplication.c' - - -@rfm.simple_test -class OpenMPTest(BaseMatrixVectorTest): - def __init__(self): - super().__init__('OpenMP') - self.sourcepath = 'example_matrix_vector_multiplication_openmp.c' - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.prgenv_flags = { - 'PrgEnv-cray': ['-homp'], - 'PrgEnv-gnu': ['-fopenmp'], - 'PrgEnv-intel': ['-openmp'], - 'PrgEnv-pgi': ['-mp'] - } - self.variables = { - 'OMP_NUM_THREADS': '4' - } - - -@rfm.simple_test -class MPITest(BaseMatrixVectorTest): - def __init__(self): - super().__init__('MPI') - self.valid_systems = ['daint:gpu', 'daint:mc'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-gnu', - 'PrgEnv-intel', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_mpi_openmp.c' - self.prgenv_flags = { - 'PrgEnv-cray': ['-homp'], - 'PrgEnv-gnu': ['-fopenmp'], - 'PrgEnv-intel': ['-openmp'], - 'PrgEnv-pgi': ['-mp'] - } - self.num_tasks = 8 - self.num_tasks_per_node = 2 - self.num_cpus_per_task = 4 - self.variables = { - 'OMP_NUM_THREADS': str(self.num_cpus_per_task) - } - - -@rfm.simple_test -class OpenACCTest(BaseMatrixVectorTest): - def __init__(self): - super().__init__('OpenACC') - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['PrgEnv-cray', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_openacc.c' - self.modules = ['craype-accel-nvidia60'] - self.num_gpus_per_node = 1 - self.prgenv_flags = { - 'PrgEnv-cray': ['-hacc', '-hnoomp'], - 'PrgEnv-pgi': ['-acc', '-ta=tesla:cc60'] - } - - -@rfm.simple_test -class CudaTest(BaseMatrixVectorTest): - def __init__(self): - super().__init__('CUDA') - self.valid_systems = ['daint:gpu'] - self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-cray', 'PrgEnv-pgi'] - self.sourcepath = 'example_matrix_vector_multiplication_cuda.cu' - self.modules = ['cudatoolkit'] - self.num_gpus_per_node = 1 diff --git a/tutorial/src/example_matrix_vector_multiplication.c b/tutorial/src/example_matrix_vector_multiplication.c deleted file mode 100644 index c366d49a00..0000000000 --- a/tutorial/src/example_matrix_vector_multiplication.c +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include - -int main (int argc, char *argv[]){ - double *matrix, *vector_in, *vector_out, out; - long dim_mn, iterations, i, j, iteration; - struct timeval start, stop, tdiff; - - if (argc!=3){ - fprintf(stderr, "%s matrixdimension numberofiterations\n", argv[0]); - exit(1); - } - dim_mn = atoi(argv[1]); - iterations = atoi(argv[2]); - - if ((dim_mn<1)||(iterations<1)){ - fprintf(stderr, "matrixdimension and numberofiterations must be " - "positive integers\n"); - exit(2); - } - - matrix = (double*) malloc(dim_mn*dim_mn*sizeof(double)); - vector_in = (double*) malloc(dim_mn*sizeof(double)); - vector_out = (double*) malloc(dim_mn*sizeof(double)); - for (i=0; i -#include -#include -#include - -int main (int argc, char *argv[]){ - double *matrix, *vector_in, *vector_out, out; - long dim_mn, iterations, i, j, iteration; - struct timeval start, stop, tdiff; - int mpi_size, mpi_rank; - double exec_time; - - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); - MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); - - if (argc!=3){ - if (mpi_rank==0){ - fprintf(stderr, "%s matrixdimension numberofiterations\n", - argv[0]); - } - exit(1); - } - dim_mn = atoi(argv[1]); - iterations = atoi(argv[2]); - - if ((dim_mn<1)||(iterations<1)){ - if (mpi_rank==0){ - fprintf(stderr, "matrixdimension and numberofiterations must be " - "positive integers\n"); - } - exit(2); - } - if (dim_mn%mpi_size!=0){ - if (mpi_rank==0){ - fprintf(stderr, "matrixdimension must be a multiple of number of " - "MPI ranks\n"); - } - exit(3); - } - - matrix = (double*) malloc(dim_mn/mpi_size*dim_mn*sizeof(double)); - vector_in = (double*) malloc(dim_mn*sizeof(double)); - vector_out = (double*) malloc(dim_mn/mpi_size*sizeof(double)); - for (i=0; i -#include -#include - -int main (int argc, char *argv[]){ - double *matrix, *vector_in, *vector_out, out; - long dim_mn, iterations, i, j, iteration; - struct timeval start, stop, tdiff; - - if (argc!=3){ - fprintf(stderr, "%s matrixdimension numberofiterations\n", argv[0]); - exit(1); - } - dim_mn = atoi(argv[1]); - iterations = atoi(argv[2]); - - if ((dim_mn<1)||(iterations<1)){ - fprintf(stderr, "matrixdimension and numberofiterations must be " - "positive integers\n"); - exit(2); - } - - matrix = (double*) malloc(dim_mn*dim_mn*sizeof(double)); - vector_in = (double*) malloc(dim_mn*sizeof(double)); - vector_out = (double*) malloc(dim_mn*sizeof(double)); - for (i=0; i\S+)', self.stdout, 'number', float) + r'Random: (?P\S+)', self.stdout, 'number', float + ) self.sanity_patterns = sn.all([ sn.assert_eq(sn.count(numbers), 100), - sn.all(sn.map(lambda x: sn.assert_bounded(x, 50, 80), numbers)), - sn.assert_found('FINISHED', self.stdout) + sn.all(sn.map(lambda x: sn.assert_bounded(x, 90, 100), numbers)), + sn.assert_found(r'FINISHED', self.stdout) ]) - self.maintainers = ['put-your-name-here'] - self.tags = {'tutorial'} diff --git a/tutorial/advanced/src/limits.sh b/tutorials/misc/random/src/limits.sh similarity index 100% rename from tutorial/advanced/src/limits.sh rename to tutorials/misc/random/src/limits.sh From 349e00e22d3821dcc7b913949f38ffa1d75918c2 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 5 Jul 2020 11:07:51 +0200 Subject: [PATCH 3/7] Add configuration for container platforms in tutorial config --- docs/config_reference.rst | 3 +++ docs/tutorial_basics.rst | 4 ++-- docs/tutorial_misc_topics.rst | 12 +++++++++++- tutorials/config/settings.py | 6 ++++++ 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docs/config_reference.rst b/docs/config_reference.rst index eaa0f5ff47..ac64b7fc28 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -313,6 +313,9 @@ System Partition Configuration +.. _container-platform-configuration: + + Container Platform Configuration ================================ diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index 44b4ceb7a5..1845046dbb 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -238,7 +238,7 @@ Note that you should *not* edit this configuration file in place. Here is how the new configuration file looks like with the needed additions highlighted: .. literalinclude:: ../tutorials/config/settings.py - :lines: 10-25,59-79,112- + :lines: 10-25,66-86,119- :emphasize-lines: 3-16,32-43 Here we define a system named ``catalina`` that has one partition named ``default``. @@ -675,7 +675,7 @@ Let's extend our configuration file for Piz Daint. .. literalinclude:: ../tutorials/config/settings.py - :lines: 10- + :lines: 10-46,53- :emphasize-lines: 17-50,72-103 diff --git a/docs/tutorial_misc_topics.rst b/docs/tutorial_misc_topics.rst index ae67b68412..3fba947ea5 100644 --- a/docs/tutorial_misc_topics.rst +++ b/docs/tutorial_misc_topics.rst @@ -323,7 +323,16 @@ Testing containerized applications ReFrame can be used also to test applications that run inside a container. -Here is a test that does so: +First, we need to enable the container platform support in ReFrame's configuration and, specifically, at the partition configuration level: + +.. literalinclude:: ../tutorials/config/settings.py + :lines: 40-54 + :emphasize-lines: 8-13 + +For each partition, users can define a list of container platforms supported using the :js:attr:`container_platforms` configuration parameter. +In this case, we define the `Singularity `__ platform, for which we set the :js:attr:`modules` parameter in order to instruct ReFrame to load the ``singularity`` module, whenever it needs to run with this container platform. + +The following test will use a Singularity container to run: .. literalinclude:: ../tutorials/misc/containers/container_test.py :lines: 6- @@ -356,3 +365,4 @@ Besides the stage directory, additional mount points can be specified through th ('/path/to/host/dir2', '/path/to/container/mount_point2')] For a complete list of the available attributes of a specific container platform, please have a look at the :ref:`container-platforms` section of the :doc:`regression_test_api` guide. +On how to configure ReFrame for running containerized tests, please have a look at the :ref:`container-platform-configuration` section of the :doc:`config_reference`. diff --git a/tutorials/config/settings.py b/tutorials/config/settings.py index 8e73eec0da..3c35fb56d5 100644 --- a/tutorials/config/settings.py +++ b/tutorials/config/settings.py @@ -44,6 +44,12 @@ 'launcher': 'srun', 'access': ['-C gpu', '-A csstaff'], 'environs': ['gnu', 'intel', 'pgi', 'cray'], + 'container_platforms': [ + { + 'type': 'Singularity', + 'modules': ['singularity'] + } + ], 'max_jobs': 100 }, { From aefeef02b1f06d3716c4d06448b52e405156ddd1 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 5 Jul 2020 11:13:06 +0200 Subject: [PATCH 4/7] Update container test --- tutorials/misc/containers/container_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tutorials/misc/containers/container_test.py b/tutorials/misc/containers/container_test.py index 39e47e751f..618cc22d61 100644 --- a/tutorials/misc/containers/container_test.py +++ b/tutorials/misc/containers/container_test.py @@ -22,6 +22,5 @@ def __init__(self): self.sanity_patterns = sn.all([ sn.assert_found(r'^' + self.container_platform.workdir, self.stdout), - sn.assert_found(r'^advanced_example1.c', self.stdout), sn.assert_found(r'18.04.\d+ LTS \(Bionic Beaver\)', self.stdout), ]) From 2c63a4cbe2db6b16a9c5c11706f1b26d7ea9d14f Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 5 Jul 2020 18:44:53 +0200 Subject: [PATCH 5/7] Fix account for CI runs. --- ci-scripts/ci-runner.bash | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 847a7bf6de..88a8743250 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -41,7 +41,7 @@ checked_exec() run_tutorial_checks() { - cmd="./bin/reframe -C tutorials/config/settings.py \ + cmd="./bin/reframe -C tutorials/config/settings.py -J account=jenscscs \ --save-log-files -r -c tutorials/ -R -x HelloThreadedExtendedTest $@" echo "[INFO] Running tutorial checks with \`$cmd'" checked_exec $cmd From ef5ca62ebdee98eb3a1214a1b5d14a0ece58bb56 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Mon, 6 Jul 2020 22:18:30 +0200 Subject: [PATCH 6/7] Address PR comments --- docs/tutorial_basics.rst | 6 +++--- docs/tutorial_misc_topics.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index 1845046dbb..7cb1972f23 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -4,7 +4,7 @@ .. versionadded:: 3.1 -This tutorial will give you a first overview of ReFrame and will acquaint with its basic concepts. +This tutorial will give you a first overview of ReFrame and will acquaint you with its basic concepts. We will start with a simple "Hello, World!" test running with the default configuration and we will expand the example along the way. We will also explore performance tests and we will port our tests to an HPC cluster. The examples of this tutorial can be found under :obj:`tutorials/basics/`. @@ -130,7 +130,7 @@ Let's inspect what files ReFrame produced for this test: rfm_HelloTest_build.err rfm_HelloTest_build.sh rfm_HelloTest_job.out rfm_HelloTest_build.out rfm_HelloTest_job.err rfm_HelloTest_job.sh -ReFrame stores in the output directory of the test the build and "job" scripts it generated for building and running the code along with their standard output and error. +ReFrame stores in the output directory of the test the build and run scripts it generated for building and running the code along with their standard output and error. All these files are prefixed with ``rfm_``. @@ -915,7 +915,7 @@ In ReFrame, you don't have to care about all the system interaction details, but Adapting a test to new systems and programming environments ----------------------------------------------------------- -Unless a test is rather generic, you will need to do some adaptations for the system that you port it to. +Unless a test is rather generic, you will need to make some adaptations for the system that you port it to. In this case, we will adapt the STREAM benchmark so as to run it with multiple compiler and adjust its execution parameters based on the target architecture of each partition. Let's see and comment the changes: diff --git a/docs/tutorial_misc_topics.rst b/docs/tutorial_misc_topics.rst index 3fba947ea5..c97c20fbe8 100644 --- a/docs/tutorial_misc_topics.rst +++ b/docs/tutorial_misc_topics.rst @@ -354,7 +354,7 @@ ReFrame will run the container as follows: singularity exec -B"/path/to/test/stagedir:/workdir" docker://ubuntu:18.04 bash -c 'cd rfm_workdir; pwd; ls; cat /etc/os-release' By default ReFrame will mount the stage directory of the test under ``/rfm_workdir`` inside the container and it will always prepend a ``cd`` command to that directory. -The user commands then are then run from that directory one after the other. +The user commands are then run from that directory one after the other. Once the commands are executed, the container is stopped and ReFrame goes on with the sanity and performance checks. Users may also change the default mount point of the stage directory by using :attr:`workdir ` attribute: Besides the stage directory, additional mount points can be specified through the :attr:`mount_points ` attribute: From ba0e0db0f3f6dbcab75d0ea3b0488fc2491e613d Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 9 Jul 2020 16:55:10 +0200 Subject: [PATCH 7/7] Address PR comments (v2) --- docs/conf.py | 4 ---- docs/configure.rst | 4 ++-- docs/tutorial_basics.rst | 16 ++++++++-------- reframe/core/pipeline.py | 2 +- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 76c757e543..6710a101d3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -144,10 +144,6 @@ 'logo_only': True } html_logo = "_static/img/reframe-logo-dark-bg.svg" - - -# autodoc_default_flags=['members', 'undoc-members', 'private-members', 'special-members', 'inherited-members', 'show-inheritance'] -# autodoc_default_flags=['members', 'undoc-members', 'inherited-members', 'show-inheritance'] autodoc_default_flags = ['members'] # End of READTHEDOCS diff --git a/docs/configure.rst b/docs/configure.rst index d4d8d39664..acb524738e 100644 --- a/docs/configure.rst +++ b/docs/configure.rst @@ -137,7 +137,7 @@ A programming environment in ReFrame is essentially a collection of environment An important feature in ReFrame's configuration, is that you can define section objects differently for different systems or system partitions by using the ``target_systems`` property. Notice, for example, how the ``gnu`` environment is defined differently for the system ``daint`` compared to the generic definition. The ``target_systems`` property is a list of systems or system/partition combinations where this definition of the environment is in effect. -This means that ``gnu`` will defined this way only for regression tests running on ``daint``. +This means that ``gnu`` will be defined this way only for regression tests running on ``daint``. For all the other systems, it will be defined using the first definition. @@ -388,4 +388,4 @@ Let's see some concrete examples: "CC" - If you explicitly query a configuration value and this value is not defined in the configuration file, ReFrame will print its default value. + If you explicitly query a configuration value which is not defined in the configuration file, ReFrame will print its default value. diff --git a/docs/tutorial_basics.rst b/docs/tutorial_basics.rst index 7cb1972f23..ed5e7cd598 100644 --- a/docs/tutorial_basics.rst +++ b/docs/tutorial_basics.rst @@ -6,7 +6,7 @@ This tutorial will give you a first overview of ReFrame and will acquaint you with its basic concepts. We will start with a simple "Hello, World!" test running with the default configuration and we will expand the example along the way. -We will also explore performance tests and we will port our tests to an HPC cluster. +We will also explore performance tests and port our tests to an HPC cluster. The examples of this tutorial can be found under :obj:`tutorials/basics/`. @@ -147,7 +147,7 @@ ReFrame allows you to avoid this in several ways but the most compact is to defi :lines: 6- -This exactly the same test as the ``hello1.py`` except that it is decorated with the :func:`@parameterized_test ` decorator instead of the :func:`@simple_test `. +This is exactly the same test as the ``hello1.py`` except that it is decorated with the :func:`@parameterized_test ` decorator instead of the :func:`@simple_test `. Also the constructor of the test now takes an argument. The :func:`@parameterized_test <>` decorator instructs ReFrame to instantiate a test class with different parameters. In this case the test will be instantiated for both C and C++ and then we use the ``lang`` parameter directly as the extension of the source file. @@ -211,7 +211,7 @@ Oops! The C++ test has failed. ReFrame complains that it does not know how to compile a C++ program. Remember our discussion above that the default configuration of ReFrame defines a minimal programming environment named ``builtin`` which only knows of a ``cc`` compiler. We will fix that in a moment, but before doing that it's worth looking into the failure information provided for the test. -For each test failing, ReFrame will print a short summary with information about the system partition and the programming environment that the test failed for, its job or process id (if any), the nodes it was running on, its stage directory, the phase that failed etc. +For each failed test, ReFrame will print a short summary with information about the system partition and the programming environment that the test failed for, its job or process id (if any), the nodes it was running on, its stage directory, the phase that failed etc. When a test fails its stage directory is kept intact, so that users can inspect the failure and try to reproduce it manually. In this case, the stage directory contains only the "Hello, World" source files, since ReFrame could not produce a build script for the C++ test, as it doesn't know to compile a C++ program for the moment. @@ -253,7 +253,7 @@ Notice, how the ``generic`` system matches any hostname, so that it acts as a fa .. note:: The different systems in the configuration file are tried in order and the first match is picked. - This practically means that the more general the selection pattern for a system is, the lower in the list of systems should be. + This practically means that the more general the selection pattern for a system is, the lower in the list of systems it should be. The :doc:`configure` page describes the configuration file in more detail and the :doc:`config_reference` provides a complete reference guide of all the configuration options of ReFrame. @@ -329,7 +329,7 @@ Here is the corresponding ReFrame test, where the new concepts introduced are hi :emphasize-lines: 11-13 ReFrame delegates the compilation of a test to a *build system*, which is an abstraction of the steps needed to compile the test. -Build system take also care of interactions with the programming environment if necessary. +Build systems take also care of interactions with the programming environment if necessary. Compilation flags are a property of the build system. If not explicitly specified, ReFrame will try to pick the correct build system (e.g., CMake, Autotools etc.) by inspecting the test resources, but in cases as the one presented here where we need to set the compilation flags, we need to specify a build system explicitly. In this example, we instruct ReFrame to compile a single source file using the ``-std=c++11 -Wall`` compilation flags. @@ -532,7 +532,7 @@ Let's run the test now: ./bin/reframe -c tutorials/basics/stream/stream1.py -r --performance-report -The :option:`--performance-report` will generated a short report at the end for each performance test that has run. +The :option:`--performance-report` will generate a short report at the end for each performance test that has run. .. code-block:: none @@ -619,7 +619,7 @@ Examining the performance logs ReFrame has a powerful mechanism for logging its activities as well as performance data. It supports different types of log channels and it can send data simultaneously in any number of them. -For examples, performance data might be logged in files and the same time being send to Syslog or to a centralized log management server. +For example, performance data might be logged in files and the same time being send to Syslog or to a centralized log management server. By default (i.e., starting off from the builtin configuration file), ReFrame sends performance data to files per test under the ``perflogs/`` directory: .. code-block:: none @@ -868,7 +868,7 @@ We will only do so with the final versions of the tests from the previous sectio There it is! Without any change in our tests, we could simply run them in a HPC cluster with all of its intricacies. -Notice how our just four tests expanded to almost 40 test cases on that particular HPC cluster! +Notice how our original four tests expanded to almost 40 test cases on that particular HPC cluster! One reason we could run immediately our tests on a new system was that we have not been restricting neither the valid system they can run nor the valid programming environments they can run with (except for the STREAM test). Otherwise we would have to add ``daint`` and its corresponding programming environments in :attr:`valid_systems` and :attr:`valid_prog_environs` lists respectively. diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 6c9e761628..85d09d7320 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -436,7 +436,7 @@ class RegressionTest(metaclass=RegressionTestMeta): int, type(None)) #: Number of GPUs per node required by this test. - #: This is attribute translated internally to the ``_rfm_gpu`` resource. + #: This attribute is translated internally to the ``_rfm_gpu`` resource. #: For more information on test resources, have a look at the #: :attr:`extra_resources` attribute. #: