diff --git a/docs/_static/img/reframe-test-cases.svg b/docs/_static/img/reframe-test-cases.svg new file mode 100644 index 0000000000..511a51fe81 --- /dev/null +++ b/docs/_static/img/reframe-test-cases.svg @@ -0,0 +1,3 @@ + + +
T0.__init__()
T0.__init__()
Filter tests
Filter tests
@rfm.simple_test
class T0(rfm.RegressionTest):
    def __init__(self):
        self.valid_systems = ['P0', 'P1']
        self.valid_prog_environs = ['E0', 'E1']
[Not supported by viewer]
(T0, P0, E0)
(T0, P0, E0)
(T0, P0, E1)
(T0, P0, E1)
(T0, P1, E0)
(T0, P1, E0)
(T0, P1, E1)
(T0, P1, E1)
(T0, P0, E0)
(T0, P0, E0)
(T0, P0, E1)
(T0, P0, E1)
(T0, P1, E0)
(T0, P1, E0)
(T0, P1, E1)
(T0, P1, E1)
Test cases
(T0 cloned)
[Not supported by viewer]
Runner
Runner
\ No newline at end of file diff --git a/docs/_static/img/test-deps-by-env.svg b/docs/_static/img/test-deps-by-env.svg new file mode 100644 index 0000000000..54b6a25cf2 --- /dev/null +++ b/docs/_static/img/test-deps-by-env.svg @@ -0,0 +1,3 @@ + + +
T0,E0
T0,E0
T0,E1
T0,E1
T1,E0
T1,E0
T1,E1
T1,E1
T0
T0
T1
T1
\ No newline at end of file diff --git a/docs/_static/img/test-deps-cycle.svg b/docs/_static/img/test-deps-cycle.svg new file mode 100644 index 0000000000..35a15fd9ff --- /dev/null +++ b/docs/_static/img/test-deps-cycle.svg @@ -0,0 +1,3 @@ + + +
T0,E0
T0,E0
T0,E1
T0,E1
T1,E0
T1,E0
T1,E1
T1,E1
T0
T0
T1
T1
X
X
\ No newline at end of file diff --git a/docs/_static/img/test-deps-dangling.svg b/docs/_static/img/test-deps-dangling.svg new file mode 100644 index 0000000000..544dc80478 --- /dev/null +++ b/docs/_static/img/test-deps-dangling.svg @@ -0,0 +1,3 @@ + + +
T0,E0
T0,E0
T1,E0
T1,E0
T1,E1
T1,E1
T0
T0
T1
T1
X
X
\ No newline at end of file diff --git a/docs/_static/img/test-deps-exact.svg b/docs/_static/img/test-deps-exact.svg new file mode 100644 index 0000000000..1ad602664e --- /dev/null +++ b/docs/_static/img/test-deps-exact.svg @@ -0,0 +1,3 @@ + + +
T0,E0
T0,E0
T0,E1
T0,E1
T1,E0
T1,E0
T1,E1
T1,E1
T0
T0
T1
T1
\ No newline at end of file diff --git a/docs/_static/img/test-deps-fully.svg b/docs/_static/img/test-deps-fully.svg new file mode 100644 index 0000000000..92fbdb8891 --- /dev/null +++ b/docs/_static/img/test-deps-fully.svg @@ -0,0 +1,3 @@ + + +
T0,E0
T0,E0
T0,E1
T0,E1
T1,E0
T1,E0
T1,E1
T1,E1
T0
T0
T1
T1
\ No newline at end of file diff --git a/docs/_static/img/test-deps.svg b/docs/_static/img/test-deps.svg new file mode 100644 index 0000000000..cfa210aa97 --- /dev/null +++ b/docs/_static/img/test-deps.svg @@ -0,0 +1,3 @@ + + +
T0
T0
T1
T1
\ No newline at end of file diff --git a/docs/advanced.rst b/docs/advanced.rst index 12d8074c4a..1dd15d089d 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -505,3 +505,180 @@ Besides the stage directory, additional mount points can be specified through th For a complete list of the available attributes of a specific container platform, the reader is referred to `reference guide `__. + + +Using dependencies in your tests +-------------------------------- + +.. versionadded:: 2.21 + +A ReFrame test may define dependencies to other tests. +An example scenario is to test different runtime configurations of a benchmark that you need to compile, or run a scaling analysis of a code. +In such cases, you don't want to rebuild your test for each runtime configuration. +You could have a build test, which all runtime tests would depend on. +This is the approach we take with the following example, that fetches, builds and runs several `OSU benchmarks `__. +We first create a basic compile-only test, that fetches the benchmarks and builds them for the different programming environments: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 87-101 + +There is nothing particular to that test, except perhaps that you can set :attr:`sourcesdir` to ``None`` even for a test that needs to compile something. +In such a case, you should at least provide the commands that fetch the code inside the :attr:`prebuild_cmd` attribute. + +For the next test we need to use the OSU benchmark binaries that we just built, so as to run the MPI ping-pong benchmark. +Here is the relevant part: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 7-39 + +First, since we will have multiple similar benchmarks, we move all the common functionality to the :class:`OSUBenchmarkTestBase` base class. +Again nothing new here; we are going to use two nodes for the benchmark and we set :attr:`sourcesdir` to ``None``, since none of the benchmark tests will use any additional resources. +The new part comes in with the :class:`OSULatencyTest` test in the following line: + + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 27 + +Here we tell ReFrame that this test depends on a test named ``OSUBuildTest``. +This test may or may not be defined in the same test file; all ReFrame needs is the test name. +By default, the :func:`depends_on` function will create dependencies between the individual test cases of the :class:`OSULatencyTest` and the :class:`OSUBuildTest`, such that the :class:`OSULatencyTest` using ``PrgEnv-gnu`` will depend on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-gnu``, but not on the outcome of the :class:`OSUBuildTest` using ``PrgEnv-intel``. +This behaviour can be changed, but we will return to this later. +You can create arbitrary test dependency graphs, but they need to be acyclic. +If ReFrame detects cyclic dependencies, it will refuse to execute the set of tests and will issue an error pointing out the cycle. + +A ReFrame test with dependencies will execute, i.e., enter its `setup` stage, only after `all` of its dependencies have succeeded. +If any of its dependencies fails, the current test will be marked as failure as well. + +The next step for the :class:`OSULatencyTest` is to set its executable to point to the binary produced by the :class:`OSUBuildTest`. +This is achieved with the following specially decorated function: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 32-38 + +The ``@require_deps`` decorator will bind the arguments passed to the decorated function to the result of the dependency that each argument names. +In this case, it binds the ``OSUBuildTest`` function argument to the result of a dependency named ``OSUBuildTest``. +In order for the binding to work correctly the function arguments must be named after the target dependencies. +However, referring to a dependency only by the test's name is not enough, since a test might be associated with multiple programming environments. +For this reason, a dependency argument is actually bound to a function that accepts as argument the name of a target programming environment. +If no arguments are passed to that function, as in this example, the current programming environment is implied, such that ``OSUBuildTest()`` is equivalent to ``OSUBuildTest(self.current_environ.name)``. +This call returns the actual test case of the dependency that has been executed. +This allows you to access any attribute from the target test, as we do in this example by accessing the target test's stage directory, which we use to construct the path of the executable. +This concludes the presentation of the :class:`OSULatencyTest` test. The :class:`OSUBandwidthTest` is completely analogous. + +The :class:`OSUAllreduceTest` shown below is similar to the other two, except that it is parameterized. +It is essentially a scalability test that is running the ``osu_allreduce`` executable created by the :class:`OSUBuildTest` for 2, 4, 8 and 16 nodes. + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 64-84 + +The full set of OSU example tests is shown below: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + +Notice that the order dependencies are defined in a test file is irrelevant. +In this case, we define :class:`OSUBuildTest` at the end. +ReFrame will make sure to properly sort the tests and execute them. + +Here is the output when running the OSU tests with the asynchronous execution policy: + +.. code-block:: none + + [==========] Running 7 check(s) + [==========] Started on Tue Dec 10 00:15:53 2019 + + [----------] started processing OSUBuildTest (OSU benchmarks build test) + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-intel + [ RUN ] OSUBuildTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUBuildTest (OSU benchmarks build test) + + [----------] started processing OSULatencyTest (OSU latency test) + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-gnu + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-intel + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-intel + [ RUN ] OSULatencyTest on daint:gpu using PrgEnv-pgi + [ DEP ] OSULatencyTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSULatencyTest (OSU latency test) + + [----------] started processing OSUBandwidthTest (OSU bandwidth test) + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-intel + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-intel + [ RUN ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [ DEP ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUBandwidthTest (OSU bandwidth test) + + [----------] started processing OSUAllreduceTest_2 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_2 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_4 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_4 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_8 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_8 (OSU Allreduce test) + + [----------] started processing OSUAllreduceTest_16 (OSU Allreduce test) + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ RUN ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [ DEP ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [----------] finished processing OSUAllreduceTest_16 (OSU Allreduce test) + + [----------] waiting for spawned checks to finish + [ OK ] OSUBuildTest on daint:gpu using PrgEnv-pgi + [ OK ] OSUBuildTest on daint:gpu using PrgEnv-gnu + [ OK ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-pgi + [ OK ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-gnu + [ OK ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-gnu + [ OK ] OSUBuildTest on daint:gpu using PrgEnv-intel + [ OK ] OSULatencyTest on daint:gpu using PrgEnv-gnu + [ OK ] OSUBandwidthTest on daint:gpu using PrgEnv-gnu + [ OK ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-gnu + [ OK ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-pgi + [ OK ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-pgi + [ OK ] OSULatencyTest on daint:gpu using PrgEnv-intel + [ OK ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-intel + [ OK ] OSUAllreduceTest_16 on daint:gpu using PrgEnv-intel + [ OK ] OSUBandwidthTest on daint:gpu using PrgEnv-pgi + [ OK ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-pgi + [ OK ] OSUAllreduceTest_8 on daint:gpu using PrgEnv-intel + [ OK ] OSUAllreduceTest_4 on daint:gpu using PrgEnv-gnu + [ OK ] OSULatencyTest on daint:gpu using PrgEnv-pgi + [ OK ] OSUAllreduceTest_2 on daint:gpu using PrgEnv-intel + [ OK ] OSUBandwidthTest on daint:gpu using PrgEnv-intel + [----------] all spawned checks have finished + + [ PASSED ] Ran 21 test case(s) from 7 check(s) (0 failure(s)) + [==========] Finished on Tue Dec 10 00:21:11 2019 + +Before starting running the tests, ReFrame topologically sorts them based on their dependencies and schedules them for running using the selected execution policy. +With the serial execution policy, ReFrame simply executes the tests to completion as they "arrive", since the tests are already topologically sorted. +In the asynchronous execution policy, tests are spawned and not waited for. +If a test's dependencies have not yet completed, it will not start its execution and a ``DEP`` message will be printed to denote this. + +Finally, ReFrame's runtime takes care of properly cleaning up the resources of the tests respecting dependencies. +Normally when an individual test finishes successfully, its stage directory is cleaned up. +However, if other tests are depending on this one, this would be catastrophic, since most probably the dependent tests would need the outcome of this test. +ReFrame fixes that by not cleaning up the stage directory of a test until all its dependent tests have finished successfully. diff --git a/docs/dependencies.rst b/docs/dependencies.rst new file mode 100644 index 0000000000..650caae85f --- /dev/null +++ b/docs/dependencies.rst @@ -0,0 +1,189 @@ +===================================== +How Test Dependencies Work In ReFrame +===================================== + +Before going into details on how ReFrame treats test dependencies, it is important to understand how tests are actually treated and executed by the runtime. +Normally, a ReFrame test will be tried for different programming environments and different partitions within the same ReFrame run. +These are defined in the test's :func:`__init__` method, but it is not this original object that is being executed by the regression test pipeline. +The following figure explains in more detail the process: + +.. figure:: _static/img/reframe-test-cases.svg + :align: center + :alt: How ReFrame loads and schedules tests for execution. + +When ReFrame loads a test from the disk it unconditionally constructs it executing its :func:`__init__` method. +The practical implication of this is that your test will be instantiated even if it will not run on the current system. +After all the tests are loaded, they are filtered based on the current system and any other criteria (such as programming environment, test attributes etc.) specified by the user (see `Filtering of Regression Tests `__ for more details). +After the tests are filtered, ReFrame creates the actual `test cases` to be run. A test case is essentially a tuple consisting of the test, the system partition and the programming environment to try. +The test that goes into a test case is essentially a `clone` of the original test that was instantiated upon loading. +This ensures that the test case's state is not shared and may not be reused in any case. +Finally, the generated test cases are passed to a `runner` that is responsible for scheduling them for execution based on the selected execution policy. + +Dependencies in ReFrame are defined at the test level using the :func:`depends_on` function, but are projected to the test cases space. +We will see the rules of that projection in a while. +The dependency graph construction and the subsequent dependency analysis happen also at the level of the test cases. + +Let's assume that test :class:`T1` depends in :class:`T0`. +This can be expressed inside :class:`T1` using the :func:`depends_on` method: + +.. code:: python + + @rfm.simple_test + class T1(rfm.RegressionTest): + def __init__(self): + ... + self.depends_on('T0') + +Conceptually, this dependency can be viewed at the test level as follows: + + +.. figure:: _static/img/test-deps.svg + :align: center + :alt: Simple test dependency presented conceptually. + +For most of the cases, this is sufficient to reason about test dependencies. +In reality, as mentioned above, dependencies are handled at the level of test cases. +Test cases on different partitions are always independent. +If not specified differently, test cases using programming environments are also independent. +This is the default behavior of the :func:`depends_on` function. +The following image shows the actual test case dependencies assuming that both tests support the ``E0`` and ``E1`` programming environments (for simplicity, we have omitted the partitions, since tests are always independent in that dimension): + +.. figure:: _static/img/test-deps-by-env.svg + :align: center + :alt: Test case dependencies by environment (default). + +This means that test cases of :class:`T1` may start executing before all test cases of :class:`T0` have finished. +You can impose a stricter dependency between tests, such that :class:`T1` does not start execution unless all test cases of :class:`T0` have finished. +You can achieve this as follows: + +.. code:: python + + @rfm.simple_test + class T1(rfm.RegressionTest): + def __init__(self): + ... + self.depends_on('T0', how=rfm.DEPEND_FULLY) + +This will create the following test case graph: + +.. figure:: _static/img/test-deps-fully.svg + :align: center + :alt: Fully dependent test cases. + + +You may also create arbitrary dependencies between the test cases of different tests, like in the following example, where the dependencies cannot be represented in any of the other two ways: + +.. figure:: _static/img/test-deps-exact.svg + :align: center + :alt: Arbitrary test cases dependencies + +These dependencies can be achieved as follows: + +.. code:: python + + @rfm.simple_test + class T1(rfm.RegressionTest): + def __init__(self): + ... + self.depends_on('T0', how=rfm.DEPEND_EXACT, + subdeps={'E0': ['E0', 'E1'], 'E1': ['E1']}) + +The ``subdeps`` argument defines the sub-dependencies between the test cases of :class:`T1` and :class:`T0` using an adjacency list representation. + + +Cyclic dependencies +------------------- + +Obviously, cyclic dependencies between test cases are not allowed. +Cyclic dependencies between tests are not allowed either, even if the test case dependency graph is acyclic. +For example, the following dependency set up is invalid: + +.. figure:: _static/img/test-deps-cycle.svg + :align: center + :alt: Any cyclic dependencies between tests are not allowed, even if the underlying test case dependencies are not forming a cycle. + +The test case dependencies here, clearly, do not form a cycle, but the edge from ``(T0, E0)`` to ``(T1, E1)`` introduces a dependency from ``T0`` to ``T1`` forming a cycle at the test level. +The reason we impose this restriction is that we wanted to keep the original processing of tests by ReFrame, where all the test cases of a test are processed before moving to the next one. +Supporting this type of dependencies would require to change substantially ReFrame's output. + + +Dangling dependencies +--------------------- + +In our discussion so far, :class:`T0` and :class:`T1` had the same valid programming environments. +What happens if they do not? +Assume, for example, that :class:`T0` and :class:`T1` are defined as follows: + +.. code:: python + + import reframe as rfm + import reframe.utility.sanity as sn + + + @rfm.simple_test + class T0(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['P0'] + self.valid_prog_environs = ['E0'] + ... + + + @rfm.simple_test + class T1(rfm.RegressionTest): + def __init__(self): + self.valid_systems = ['P0'] + self.valid_prog_environs = ['E0', 'E1'] + self.depends_on('T0') + ... + +As discussed previously, :func:`depends_on` will create one-to-one dependencies between the different programming environment test cases. +So in this case it will try to create an edge from ``(T1, E1)`` to ``(T0, E1)`` as shown below: + +.. figure:: _static/img/test-deps-dangling.svg + :align: center + :alt: When the target test is valid for less programming environments than the source test, a dangling dependency would be created. + + +This edge cannot be resolved since the target test case does not exist. +ReFrame will complain and issue an error while trying to build the test dependency graph. +The remedy to this is to use either ``DEPEND_FULLY`` or pass the exact dependencies with ``DEPEND_EXACT`` to :func:`depends_on`. + +If :class:`T0` and :class:`T1` had their :attr:`valid_prog_environs` swapped, such that :class:`T0` supported ``E0`` and ``E1`` and :class:`T1` supported only ``E0``, +the default :func:`depends_on` mode would work fine. +The ``(T0, E1)`` test case would simply have no dependent test cases. + + +Resolving dependencies +---------------------- + +As shown in the `tutorial `__, test dependencies would be of limited usage if you were not able to use the results or information of the target tests. +Let's reiterate over the :func:`set_executable` function of the :class:`OSULatencyTest` that we presented previously: + +.. literalinclude:: ../tutorial/advanced/osu/osu_benchmarks.py + :lines: 32-38 + +The ``@require_deps`` decorator does some magic -- we will unravel this shortly -- with the function arguments of the :func:`set_executable` function and binds them to the target test dependencies by their name. +However, as discussed in this section, dependencies are defined at test case level, so the ``OSUBuildTest`` function argument is bound to a special function that allows you to retrieve an actual test case of the target dependency. +This is why you need to "call" ``OSUBuildTest`` in order to retrieve the desired test case. +When no arguments are passed, this will retrieve the test case corresponding to the current partition and the current programming environment. +We could always retrieve the ``PrgEnv-gnu`` case by writing ``OSUBuildTest('PrgEnv-gnu')``. +If a dependency cannot be resolved, because it is invalid, a runtime error will be thrown with an appropriate message. + +The low-level method for retrieving a dependency is the :func:`getdep() ` method of the :class:`RegressionTest`. +In fact, you can rewrite :func:`set_executable` function as follows: + +.. code:: python + + @rfm.run_after('setup') + def set_executable(self): + target = self.getdep('OSUBuildTest') + self.executable = os.path.join( + target.stagedir, + 'osu-micro-benchmarks-5.6.2', 'mpi', 'pt2pt', 'osu_latency' + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + +Now it's easier to understand what the ``@require_deps`` decorator does behind the scenes. +It binds the function arguments to a partial realization of the :func:`getdep` function and attaches the decorated function as an after-setup hook. +In fact, any ``@require_deps``-decorated function will be invoked before any other after-setup hook. diff --git a/docs/index.rst b/docs/index.rst index 0b9a551f2a..1a4fa53daa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -53,6 +53,7 @@ Publications The Regression Test Pipeline ReFrame Tutorial Customizing Further A Regression Test + How Test Dependencies Work in ReFrame Understanding The Mechanism Of Sanity Functions Running ReFrame Use cases diff --git a/docs/reference.rst b/docs/reference.rst index 81d516e3a1..f34fd9c9f1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -10,53 +10,6 @@ Internal data structures and APIs are covered only to the extent that might be h Regression test classes and related utilities --------------------------------------------- -.. class:: reframe.RegressionTest(name=None, prefix=None) - - This is an alias of :class:`reframe.core.pipeline.RegressionTest`. - - .. versionadded:: 2.13 - - -.. class:: reframe.RunOnlyRegressionTest(*args, **kwargs) - - This is an alias of :class:`reframe.core.pipeline.RunOnlyRegressionTest`. - - .. versionadded:: 2.13 - - -.. class:: reframe.CompileOnlyRegressionTest(*args, **kwargs) - - This is an alias of :class:`reframe.core.pipeline.CompileOnlyRegressionTest`. - - .. versionadded:: 2.13 - - -.. py:decorator:: reframe.run_after - - This is an alias of :func:`reframe.core.decorators.run_after`. - - .. versionadded:: 2.20 - -.. py:decorator:: reframe.run_before - - This is an alias of :func:`reframe.core.decorators.run_before`. - - .. versionadded:: 2.20 - -.. py:decorator:: reframe.simple_test - - This is an alias of :func:`reframe.core.decorators.simple_test`. - - .. versionadded:: 2.13 - - -.. py:decorator:: reframe.parameterized_test(inst=[]) - - This is an alias of :func:`reframe.core.decorators.parameterized_test`. - - .. versionadded:: 2.13 - - .. automodule:: reframe.core.decorators :members: :show-inheritance: diff --git a/reframe/core/decorators.py b/reframe/core/decorators.py index eebf20db10..182aca7217 100644 --- a/reframe/core/decorators.py +++ b/reframe/core/decorators.py @@ -207,6 +207,26 @@ def run_after(stage): def require_deps(func): + '''Denote that the decorated test method will use the test dependencies. + + The arguments of the decorated function must be named after the + dependencies that the function intends to use. The decorator will bind the + arguments to a partial realization of the + :func:`reframe.core.pipeline.RegressionTest.getdep` function, such that + conceptually the new function arguments will be the following: + + .. code:: python + + new_arg = functools.partial(getdep, orig_arg_name) + + The converted arguments are essentially functions accepting a single + argument, which is the target test's programming environment. + + This decorator is also directly available under the :mod:`reframe` module. + + .. versionadded:: 2.21 + + ''' tests = inspect.getfullargspec(func).args[1:] func._rfm_resolve_deps = True diff --git a/reframe/core/pipeline.py b/reframe/core/pipeline.py index 93d6d3b45c..6db7a1a70a 100644 --- a/reframe/core/pipeline.py +++ b/reframe/core/pipeline.py @@ -36,8 +36,27 @@ # Dependency kinds + +#: Constant to be passed as the ``how`` argument of the +#: :func:`RegressionTest.depends_on` method. It denotes that test case +#: dependencies will be explicitly specified by the user. +#: +#: This constant is directly available under the :mod:`reframe` module. DEPEND_EXACT = 1 + +#: Constant to be passed as the ``how`` argument of the +#: :func:`RegressionTest.depends_on` method. It denotes that the test cases of +#: the current test will depend only on the corresponding test cases of the +#: target test that use the same programming environment. +#: +#: This constant is directly available under the :mod:`reframe` module. DEPEND_BY_ENV = 2 + +#: Constant to be passed as the ``how`` argument of the +#: :func:`RegressionTest.depends_on` method. It denotes that each test case of +#: this test depends on all the test cases of the target test. +#: +#: This constant is directly available under the :mod:`reframe` module. DEPEND_FULLY = 3 @@ -1354,6 +1373,36 @@ def user_deps(self): return util.SequenceView(self._userdeps) def depends_on(self, target, how=DEPEND_BY_ENV, subdeps=None): + '''Add a dependency to ``target`` in this test. + + :arg target: The name of the target test. + :arg how: How the dependency should be mapped in the test cases space. + This argument can accept any of the three constants + :attr:`DEPEND_EXACT`, :attr:`DEPEND_BY_ENV` (default), + :attr:`DEPEND_FULLY`. + + :arg subdeps: An adjacency list representation of how this test's test + cases depend on those of the target test. This is only relevant if + ``how == DEPEND_EXACT``. The value of this argument is a + dictionary having as keys the names of this test's supported + programming environments. The values are lists of the programming + environments names of the target test that this test's test cases + will depend on. In the following example, this test's ``E0`` + programming environment case will depend on both ``E0`` and ``E1`` + test cases of the target test ``T0``, but its ``E1`` case will + depend only on the ``E1`` test case of ``T0``: + + .. code:: python + + self.depends_on('T0', how=rfm.DEPEND_EXACT, + subdeps={'E0': ['E0', 'E1'], 'E1': ['E1']}) + + For more details on how test dependencies work in ReFrame, please + refer to `How Test Dependencies Work In ReFrame `__. + + .. versionadded:: 2.21 + + ''' if not isinstance(target, str): raise TypeError("target argument must be of type: `str'") @@ -1368,6 +1417,20 @@ def depends_on(self, target, how=DEPEND_BY_ENV, subdeps=None): self._userdeps.append((target, how, subdeps)) def getdep(self, target, environ=None): + '''Retrieve the test case of a target dependency. + + This is a low-level method. The :func:`@require_deps + ` decorators should be + preferred. + + :arg target: The name of the target dependency to be retrieved. + :arg environ: The name of the programming environment that will be + used to retrieve the test case of the target test. If ``None``, + :attr:`RegressionTest.current_environ` will be used. + + .. versionadded:: 2.21 + + ''' if self.current_environ is None: raise DependencyError( 'cannot resolve dependencies before the setup phase' diff --git a/tutorial/advanced/osu/osu_benchmarks.py b/tutorial/advanced/osu/osu_benchmarks.py new file mode 100644 index 0000000000..f812f22eaf --- /dev/null +++ b/tutorial/advanced/osu/osu_benchmarks.py @@ -0,0 +1,101 @@ +import os + +import reframe as rfm +import reframe.utility.sanity as sn + + +class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): + '''Base class of OSU benchmarks runtime tests''' + + def __init__(self): + self.valid_systems = ['daint:gpu'] + self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-pgi', 'PrgEnv-intel'] + self.sourcesdir = None + self.num_tasks = 2 + self.num_tasks_per_node = 1 + self.sanity_patterns = sn.assert_found(r'^8', self.stdout) + + +@rfm.simple_test +class OSULatencyTest(OSUBenchmarkTestBase): + def __init__(self): + super().__init__() + self.descr = 'OSU latency test' + self.perf_patterns = { + 'latency': sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) + } + self.depends_on('OSUBuildTest') + self.reference = { + '*': {'latency': (0, None, None, 'us')} + } + + @rfm.require_deps + def set_executable(self, OSUBuildTest): + self.executable = os.path.join( + OSUBuildTest().stagedir, + 'osu-micro-benchmarks-5.6.2', 'mpi', 'pt2pt', 'osu_latency' + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + +@rfm.simple_test +class OSUBandwidthTest(OSUBenchmarkTestBase): + def __init__(self): + super().__init__() + self.descr = 'OSU bandwidth test' + self.perf_patterns = { + 'bandwidth': sn.extractsingle(r'^4194304\s+(\S+)', + self.stdout, 1, float) + } + self.depends_on('OSUBuildTest') + self.reference = { + '*': {'bandwidth': (0, None, None, 'MB/s')} + } + + @rfm.require_deps + def set_executable(self, OSUBuildTest): + self.executable = os.path.join( + OSUBuildTest().stagedir, + 'osu-micro-benchmarks-5.6.2', 'mpi', 'pt2pt', 'osu_bw' + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + +@rfm.parameterized_test(*([1 << i] for i in range(1, 5))) +class OSUAllreduceTest(OSUBenchmarkTestBase): + def __init__(self, num_tasks): + super().__init__() + self.descr = 'OSU Allreduce test' + self.perf_patterns = { + 'latency': sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) + } + self.depends_on('OSUBuildTest') + self.reference = { + '*': {'latency': (0, None, None, 'us')} + } + self.num_tasks = num_tasks + + @rfm.require_deps + def set_executable(self, OSUBuildTest): + self.executable = os.path.join( + OSUBuildTest().stagedir, + 'osu-micro-benchmarks-5.6.2', 'mpi', 'collective', 'osu_allreduce' + ) + self.executable_opts = ['-m', '8', '-x', '1000', '-i', '20000'] + + +@rfm.simple_test +class OSUBuildTest(rfm.CompileOnlyRegressionTest): + def __init__(self): + self.descr = 'OSU benchmarks build test' + self.valid_systems = ['daint:gpu'] + self.valid_prog_environs = ['PrgEnv-gnu', 'PrgEnv-pgi', 'PrgEnv-intel'] + self.sourcesdir = None + self.prebuild_cmd = [ + 'wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-5.6.2.tar.gz', + 'tar xzf osu-micro-benchmarks-5.6.2.tar.gz', + 'cd osu-micro-benchmarks-5.6.2' + ] + self.build_system = 'Autotools' + self.build_system.max_concurrency = 8 + self.sanity_patterns = sn.assert_not_found('error', self.stderr)