New style checks

ci-scripts/ci-runner.bash

@@ -4,11 +4,11 @@ scriptname=`basename $0`
 CI_FOLDER=""
 CI_GENERIC=0
 CI_TUTORIAL=0
+CI_EXITCODE=0
 TERM="${TERM:-xterm}"
 PROFILE=""
 MODULEUSE=""
 
-CI_EXITCODE=0
 
 #
 # This function prints the script usage form
@@ -41,41 +41,29 @@ checked_exec()
 
 run_tutorial_checks()
 {
-    cmd="./bin/reframe --exec-policy=async --save-log-files -r -t tutorial $@"
+    cmd="./bin/reframe -C tutorial/config/settings.py --exec-policy=async \
+--save-log-files -r -t tutorial $@"
     echo "Running tutorial checks with \`$cmd'"
     checked_exec $cmd
 }
 
 run_user_checks()
 {
-    cmd="./bin/reframe --exec-policy=async --save-log-files -r -t production $@"
+    cmd="./bin/reframe -C config/cscs.py --exec-policy=async --save-log-files \
+-r -t production $@"
     echo "Running user checks with \`$cmd'"
     checked_exec $cmd
 }
 
 run_serial_user_checks()
 {
-    cmd="./bin/reframe --exec-policy=serial --save-log-files -r -t production-serial $@"
+    cmd="./bin/reframe -C config/cscs.py --exec-policy=serial --save-log-files \
+-r -t production-serial $@"
     echo "Running user checks with \`$cmd'"
     checked_exec $cmd
 }
 
 
-save_settings()
-{
-    tempfile=$(mktemp)
-    cp reframe/settings.py $tempfile
-    echo $tempfile
-}
-
-restore_settings()
-{
-    saved=$1
-    cp $saved reframe/settings.py
-    /bin/rm $saved
-}
-
-
 ### Main script ###
 
 shortopts="h,g,t,f:,i:,l:,m:"
@@ -167,9 +155,6 @@ if [ $CI_GENERIC -eq 1 ]; then
         grep -- '--- Logging error ---'
 elif [ $CI_TUTORIAL -eq 1 ]; then
     # Run tutorial checks
-    settings_orig=$(save_settings)
-    cp tutorial/config/settings.py reframe/settings.py
-
     # Find modified or added tutorial checks
     tutorialchecks=( $(git log --name-status --oneline --no-merges -1 | \
                    awk '/^[AM]/ { print $2 } /^R0[0-9][0-9]/ { print $3 }' | \
@@ -190,17 +175,13 @@ elif [ $CI_TUTORIAL -eq 1 ]; then
             run_tutorial_checks ${tutorialchecks_path} ${invocations[i]}
         done
     fi
-
-    restore_settings $settings_orig
 else
     # Performing the unittests
     echo "=================="
     echo "Running unit tests"
     echo "=================="
-    settings_orig=$(save_settings)
-    cp config/cscs.py reframe/settings.py
 
-    checked_exec ./test_reframe.py
+    checked_exec ./test_reframe.py --rfm-user-config=config/cscs.py
 
     # Find modified or added user checks
     userchecks=( $(git log --name-status --oneline --no-merges -1 | \
@@ -226,7 +207,5 @@ else
             run_serial_user_checks ${userchecks_path} ${invocations[i]}
         done
     fi
-
-    restore_settings $settings_orig
 fi
 exit $CI_EXITCODE

cscs-checks/prgenv/environ_check.py

@@ -1,7 +1,7 @@
 import os
 
-from reframe.core.modules import get_modules_system
 from reframe.core.pipeline import RunOnlyRegressionTest
+from reframe.core.runtime import runtime
 
 
 class DefaultPrgEnvCheck(RunOnlyRegressionTest):
@@ -31,7 +31,7 @@ def wait(self):
         pass
 
     def check_sanity(self):
-        return get_modules_system().is_module_loaded('PrgEnv-cray')
+        return runtime().modules_system.is_module_loaded('PrgEnv-cray')
 
     def cleanup(self, remove_files=False, unload_env=True):
         pass
@@ -53,7 +53,8 @@ def __init__(self, **kwargs):
         self.tags = {'production'}
 
     def check_sanity(self):
-        return get_modules_system().is_module_loaded(self.current_environ.name)
+        return runtime().modules_system.is_module_loaded(
+            self.current_environ.name)
 
 
 def _get_checks(**kwargs):

docs/advanced.rst

@@ -43,7 +43,7 @@ The contents of this regression test are the following (``tutorial/advanced/adva
 The important bit here is the ``compile()`` method.
 
 .. literalinclude:: ../tutorial/advanced/advanced_example1.py
-  :lines: 21-23
+  :lines: 18-20
   :dedent: 4
 
 As in the simple single source file examples we showed in the `tutorial <tutorial.html>`__, we use the current programming environment's flags for modifying the compilation.
@@ -162,7 +162,7 @@ This ensures that the environment of the test is also set correctly at runtime.
 Finally, as already mentioned `previously <#leveraging-makefiles>`__, since the ``Makefile`` name is not one of the standard ones, it has to be passed as an argument to the :func:`compile <reframe.core.pipeline.RegressionTest.compile>` method of the base :class:`RegressionTest <reframe.core.pipeline.RegressionTest>` class as follows:
 
 .. literalinclude:: ../tutorial/advanced/advanced_example4.py
-  :lines: 24
+  :lines: 21
   :dedent: 8
 
 Setting a Time Limit for Regression Tests
@@ -175,18 +175,17 @@ The following example (``tutorial/advanced/advanced_example5.py``) demonstrates
 
 The important bit here is the following line that sets the time limit for the test to one minute:
 
-
 .. literalinclude:: ../tutorial/advanced/advanced_example5.py
-  :lines: 16
+  :lines: 13
   :dedent: 8
 
 The :attr:`time_limit <reframe.core.pipeline.RegressionTest.time_limit>` attribute is a three-tuple in the form ``(HOURS, MINUTES, SECONDS)``.
 Time limits are implemented for all the scheduler backends.
 
-The sanity condition for this test verifies that associated job has been canceled due to the time limit.
+The sanity condition for this test verifies that associated job has been canceled due to the time limit (note that this message is SLURM-specific).
 
 .. literalinclude:: ../tutorial/advanced/advanced_example5.py
-  :lines: 19-20
+  :lines: 16-17
   :dedent: 8
 
 Applying a sanity function iteratively
@@ -212,7 +211,7 @@ The contents of the ReFrame regression test contained in ``advanced_example6.py`
 First the random numbers are extracted through the :func:`extractall <reframe.utility.sanity.extractall>` function as follows:
 
 .. literalinclude:: ../tutorial/advanced/advanced_example6.py
-  :lines: 16-17
+  :lines: 14-15
   :dedent: 8
 
 The ``numbers`` variable is a deferred iterable, which upon evaluation will return all the extracted numbers.
@@ -229,7 +228,7 @@ Note that the ``and`` operator is not deferrable and will trigger the evaluation
 The full syntax for the :attr:`sanity_patterns` is the following:
 
 .. literalinclude:: ../tutorial/advanced/advanced_example6.py
-  :lines: 18-20
+  :lines: 16-18
   :dedent: 8
 
 Customizing the Generated Job Script
@@ -286,4 +285,72 @@ The parallel launch itself consists of three parts:
 #. the regression test executable as specified in the :attr:`executable <reframe.core.pipeline.RegressionTest.executable>` attribute and
 #. the options to be passed to the executable as specified in the :attr:`executable_opts <reframe.core.pipeline.RegressionTest.executable_opts>` attribute.
 
-A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against inside it.
+A key thing to note about the generated job script is that ReFrame submits it from the stage directory of the test, so that all relative paths are resolved against it.
+
+
+Working with parameterized tests
+--------------------------------
+
+.. versionadded:: 2.13
+
+We have seen already in the `basic tutorial <tutorial.html#combining-it-all-together>`__ how we could better organize the tests so as to avoid code duplication by using test class hierarchies.
+An alternative technique, which could also be used in parallel with the class hierarchies, is to use `parameterized tests`.
+The following is a test that takes a ``variant`` parameter, which controls which variant of the code will be used.
+Depending on that value, the test is set up differently:
+
+.. literalinclude:: ../tutorial/advanced/advanced_example8.py
+
+If you have already gone through the `tutorial <tutorial.html>`__, this test can be easily understood.
+The new bit here is the ``@parameterized_test`` decorator of the ``MatrixVectorTest`` class.
+This decorator takes as argument an iterable of either sequence (i.e., lists, tuples etc.) or mapping types (i.e., dictionaries).
+Each of this iterable's elements corresponds to the arguments that will be used to instantiate the decorated test each time.
+In the example shown, the test will instantiated twice, once passing ``variant`` as ``MPI`` and a second time with ``variant`` passed as ``OpenMP``.
+The framework will try to generate unique names for the generated tests by stringifying the arguments passed to the test's constructor:
+
+
+.. code-block:: none
+
+   Command line: ./bin/reframe -C tutorial/config/settings.py -c tutorial/advanced/advanced_example8.py -l
+   Reframe version: 2.13-dev0
+   Launched by user: XXX
+   Launched on host: daint101
+   Reframe paths
+   =============
+       Check prefix      :
+       Check search path : 'tutorial/advanced/advanced_example8.py'
+       Stage dir prefix  : current/working/dir/reframe/stage/
+       Output dir prefix : current/working/dir/reframe/output/
+       Logging dir       : current/working/dir/reframe/logs
+   List of matched checks
+   ======================
+     * MatrixVectorTest_MPI (Matrix-vector multiplication test (MPI))
+           tags: [tutorial], maintainers: [you-can-type-your-email-here]
+     * MatrixVectorTest_OpenMP (Matrix-vector multiplication test (OpenMP))
+           tags: [tutorial], maintainers: [you-can-type-your-email-here]
+   Found 2 check(s).
+
+
+There are a couple of different ways that we could have used the ``@parameterized_test`` decorator.
+One is to use dictionaries for specifying the instantiations of our test class.
+The dictionaries will be converted to keyword arguments and passed to the constructor of the test class:
+
+.. code-block:: python
+
+   @rfm.parameterized_test([{'variant': 'MPI'}, {'variant': 'OpenMP'}])
+
+
+Another way, which is quite useful if you want to generate lots of different tests at the same time, is to use either `list comprehensions <https://docs.python.org/3.6/tutorial/datastructures.html#list-comprehensions>`__ or `generator expressions <https://www.python.org/dev/peps/pep-0289/>`__ for specifying the different test instantiations:
+
+.. code-block:: python
+
+   @rfm.parameterized_test((variant,) for variant in ['MPI', 'OpenMP'])
+
+
+.. note::
+   In versions of the framework prior to 2.13, this could be achieved by explicitly instantiating your tests inside the ``_get_checks()`` method.
+
+
+.. tip::
+
+   Combining parameterized tests and test class hierarchies can offer you a very flexible way for generating multiple related tests at once keeping at the same time the maintenance cost low.
+   We use this technique extensively in our tests.

docs/reference.rst

@@ -1,10 +1,62 @@
+===============
 Reference Guide
----------------
+===============
+
+This page provides a reference guide of the ReFrame API for writing regression tests covering all the relevant details.
+Internal data structures and APIs are covered only to the extent that might be helpful to the final user of the framework.
+
+
+
+Regression test classes and related utilities
+---------------------------------------------
+
+.. class:: reframe.RegressionTest(name=None, prefix=None)
+
+   This is an alias of :class:`reframe.core.pipeline.RegressionTest`.
+
+   .. versionadded:: 2.13
+
+
+.. class:: reframe.RunOnlyRegressionTest(*args, **kwargs)
+
+   This is an alias of :class:`reframe.core.pipeline.RunOnlyRegressionTest`.
+
+   .. versionadded:: 2.13
+
+
+.. class:: reframe.CompileOnlyRegressionTest(*args, **kwargs)
+
+   This is an alias of :class:`reframe.core.pipeline.CompileOnlyRegressionTest`.
+
+   .. versionadded:: 2.13
+
+
+.. py:decorator:: reframe.simple_test
+
+   This is an alias of :func:`reframe.core.decorators.simple_test`.
+
+   .. versionadded:: 2.13
+
+
+.. py:decorator:: reframe.parameterized_test(inst=[])
+
+   This is an alias of :func:`reframe.core.decorators.parameterized_test`.
+
+   .. versionadded:: 2.13
+
+
+.. automodule:: reframe.core.decorators
+   :members:
+   :show-inheritance:
 
 .. automodule:: reframe.core.pipeline
    :members:
    :show-inheritance:
 
+
+Environments and Systems
+------------------------
+
 .. automodule:: reframe.core.environments
    :members:
    :show-inheritance:
@@ -13,6 +65,10 @@ Reference Guide
    :members:
    :show-inheritance:
 
+
+Job schedulers and parallel launchers
+-------------------------------------
+
 .. autoclass:: reframe.core.schedulers.Job
    :members:
    :show-inheritance:
@@ -21,6 +77,22 @@ Reference Guide
    :members:
    :show-inheritance:
 
+
 .. automodule:: reframe.core.launchers.registry
    :members:
    :show-inheritance:
+
+
+Runtime services
+----------------
+
+.. automodule:: reframe.core.runtime
+   :members:
+   :exclude-members: temp_runtime, switch_runtime
+   :show-inheritance:
+
+
+Modules System API
+------------------
+
+.. autoclass:: reframe.core.modules.ModulesSystem