From 4e6687108fc2d079f841cd038a959a5749d31c1b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sat, 30 Oct 2021 23:53:17 +0200 Subject: [PATCH 1/7] WIP: OSU benchmarks using fixtures --- tutorials/fixtures/osu_benchmarks.py | 64 ++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tutorials/fixtures/osu_benchmarks.py diff --git a/tutorials/fixtures/osu_benchmarks.py b/tutorials/fixtures/osu_benchmarks.py new file mode 100644 index 0000000000..e814325740 --- /dev/null +++ b/tutorials/fixtures/osu_benchmarks.py @@ -0,0 +1,64 @@ +import os +import reframe as rfm +import reframe.utility.sanity as sn + + +class OSUDownloadTest(rfm.RunOnlyRegressionTest): + osu_version = variable(str, '5.6.2') + executable = f'wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{osu_version}.tar.gz' # noqa: E501 + + @sanity_function + def validate_download(self): + return sn.assert_eq(self.job.exitcode, 0) + + +class OSUBuildTest(rfm.CompileOnlyRegressionTest): + build_system = 'Autotools' + osu_tarball = fixture(OSUDownloadTest, scope='session') + + @run_before('compile') + def prepare_build(self): + tarball = f'osu-micro-benchmarks-{self.osu_tarball.osu_version}.tar.gz' + fullpath = os.path.join(self.osu_tarball.stagedir, tarball) + + self.prebuild_cmds = [ + f'cp {fullpath} {self.stagedir}', + f'tar xzf {tarball}' + ] + self.build_system.max_concurrency = 8 + + @sanity_function + def validate_build(self): + return sn.assert_not_found('error', self.stderr) + + +class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): + '''Base class of OSU benchmarks runtime tests''' + + valid_systems = ['daint:gpu'] + valid_prog_environs = ['gnu', 'pgi', 'intel'] + sourcesdir = None + num_tasks = 2 + num_tasks_per_node = 1 + osu_binaries = fixture(OSUBuildTest, scope='environment') + + @sanity_function + def validate_test(self): + return sn.assert_found(r'^8', self.stdout) + + +@rfm.simple_test +class OSULatencyTest(OSUBenchmarkTestBase): + descr = 'OSU latency test' + + @require_deps + def set_executable(self, OSUBuildTest): + self.executable = os.path.join( + osu_binaries.stagedir + 'mpi', 'pt2pt', 'osu_latency' + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + @performance_function('us') + def latency(self): + return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) From 30ac83c94cb8ae9fc408986e0b6e12b578d8a02b Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 31 Oct 2021 21:58:26 +0100 Subject: [PATCH 2/7] Better sanity check for OSU benchmarks tutorial --- tutorials/deps/osu_benchmarks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tutorials/deps/osu_benchmarks.py b/tutorials/deps/osu_benchmarks.py index 256e6881b3..0965afbba3 100644 --- a/tutorials/deps/osu_benchmarks.py +++ b/tutorials/deps/osu_benchmarks.py @@ -127,4 +127,4 @@ class OSUDownloadTest(rfm.RunOnlyRegressionTest): @sanity_function def validate_download(self): - return sn.assert_not_found('error', self.stderr) + return sn.assert_true(os.path.exists('osu-micro-benchmarks-5.6.2')) From a178bb0ef019d09d3cd6920d07816bd648e66089 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 31 Oct 2021 01:19:15 +0200 Subject: [PATCH 3/7] Rewrite OSU benchmarks using fixtures --- tutorials/fixtures/osu_benchmarks.py | 79 +++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 14 deletions(-) diff --git a/tutorials/fixtures/osu_benchmarks.py b/tutorials/fixtures/osu_benchmarks.py index e814325740..270ede659e 100644 --- a/tutorials/fixtures/osu_benchmarks.py +++ b/tutorials/fixtures/osu_benchmarks.py @@ -3,33 +3,41 @@ import reframe.utility.sanity as sn -class OSUDownloadTest(rfm.RunOnlyRegressionTest): - osu_version = variable(str, '5.6.2') - executable = f'wget http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{osu_version}.tar.gz' # noqa: E501 +class fetch_osu_benchmarks(rfm.RunOnlyRegressionTest): + version = variable(str, value='5.6.2') + executable = 'wget' + executable_opts = [ + f'http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{version}.tar.gz' # noqa: E501 + ] @sanity_function def validate_download(self): return sn.assert_eq(self.job.exitcode, 0) -class OSUBuildTest(rfm.CompileOnlyRegressionTest): +class build_osu_benchmarks(rfm.CompileOnlyRegressionTest): build_system = 'Autotools' - osu_tarball = fixture(OSUDownloadTest, scope='session') + build_prefix = variable(str) + osu_benchmarks = fixture(fetch_osu_benchmarks, scope='session') @run_before('compile') def prepare_build(self): - tarball = f'osu-micro-benchmarks-{self.osu_tarball.osu_version}.tar.gz' - fullpath = os.path.join(self.osu_tarball.stagedir, tarball) + tarball = f'osu-micro-benchmarks-{self.osu_benchmarks.version}.tar.gz' + self.build_prefix = tarball[:-7] # remove .tar.gz extension + fullpath = os.path.join(self.osu_benchmarks.stagedir, tarball) self.prebuild_cmds = [ f'cp {fullpath} {self.stagedir}', - f'tar xzf {tarball}' + f'tar xzf {tarball}', + f'cd {self.build_prefix}' ] self.build_system.max_concurrency = 8 @sanity_function def validate_build(self): - return sn.assert_not_found('error', self.stderr) + # If compilation fails, the test would fail in any case, so nothing to + # further validate here. + return True class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): @@ -40,7 +48,7 @@ class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): sourcesdir = None num_tasks = 2 num_tasks_per_node = 1 - osu_binaries = fixture(OSUBuildTest, scope='environment') + osu_binaries = fixture(build_osu_benchmarks, scope='environment') @sanity_function def validate_test(self): @@ -48,13 +56,14 @@ def validate_test(self): @rfm.simple_test -class OSULatencyTest(OSUBenchmarkTestBase): +class osu_latency_test(OSUBenchmarkTestBase): descr = 'OSU latency test' - @require_deps - def set_executable(self, OSUBuildTest): + @run_before('run') + def prepare_run(self): self.executable = os.path.join( - osu_binaries.stagedir + self.osu_binaries.stagedir, + self.osu_binaries.build_prefix, 'mpi', 'pt2pt', 'osu_latency' ) self.executable_opts = ['-x', '100', '-i', '1000'] @@ -62,3 +71,45 @@ def set_executable(self, OSUBuildTest): @performance_function('us') def latency(self): return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) + + +@rfm.simple_test +class osu_bandwidth_test(OSUBenchmarkTestBase): + descr = 'OSU bandwidth test' + + @run_before('run') + def prepare_run(self): + self.executable = os.path.join( + self.osu_binaries.stagedir, + self.osu_binaries.build_prefix, + 'mpi', 'pt2pt', 'osu_bw' + ) + self.executable_opts = ['-x', '100', '-i', '1000'] + + @performance_function('MB/s') + def bandwidth(self): + return sn.extractsingle(r'^4194304\s+(\S+)', + self.stdout, 1, float) + + +@rfm.simple_test +class osu_allreduce_test(OSUBenchmarkTestBase): + mpi_tasks = parameter(1 << i for i in range(1, 5)) + descr = 'OSU Allreduce test' + + @run_after('init') + def set_num_tasks(self): + self.num_tasks = self.mpi_tasks + + @run_before('run') + def set_executable(self): + self.executable = os.path.join( + self.osu_binaries.stagedir, + self.osu_binaries.build_prefix, + 'mpi', 'collective', 'osu_allreduce' + ) + self.executable_opts = ['-m', '8', '-x', '1000', '-i', '20000'] + + @performance_function('us') + def latency(self): + return sn.extractsingle(r'^8\s+(\S+)', self.stdout, 1, float) From 8eeffb8aade1eb45049132229134bfabef3e00ee Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 31 Oct 2021 21:59:24 +0100 Subject: [PATCH 4/7] Fine tune OSU benchmarks with fixtures --- tutorials/fixtures/osu_benchmarks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tutorials/fixtures/osu_benchmarks.py b/tutorials/fixtures/osu_benchmarks.py index 270ede659e..ff8a56d15f 100644 --- a/tutorials/fixtures/osu_benchmarks.py +++ b/tutorials/fixtures/osu_benchmarks.py @@ -4,11 +4,13 @@ class fetch_osu_benchmarks(rfm.RunOnlyRegressionTest): + descr = 'Fetch OSU benchmarks' version = variable(str, value='5.6.2') executable = 'wget' executable_opts = [ f'http://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-{version}.tar.gz' # noqa: E501 ] + local = True @sanity_function def validate_download(self): @@ -16,6 +18,7 @@ def validate_download(self): class build_osu_benchmarks(rfm.CompileOnlyRegressionTest): + descr = 'Build OSU benchmarks' build_system = 'Autotools' build_prefix = variable(str) osu_benchmarks = fixture(fetch_osu_benchmarks, scope='session') @@ -45,7 +48,6 @@ class OSUBenchmarkTestBase(rfm.RunOnlyRegressionTest): valid_systems = ['daint:gpu'] valid_prog_environs = ['gnu', 'pgi', 'intel'] - sourcesdir = None num_tasks = 2 num_tasks_per_node = 1 osu_binaries = fixture(build_osu_benchmarks, scope='environment') @@ -97,12 +99,9 @@ class osu_allreduce_test(OSUBenchmarkTestBase): mpi_tasks = parameter(1 << i for i in range(1, 5)) descr = 'OSU Allreduce test' - @run_after('init') - def set_num_tasks(self): - self.num_tasks = self.mpi_tasks - @run_before('run') def set_executable(self): + self.num_tasks = self.mpi_tasks self.executable = os.path.join( self.osu_binaries.stagedir, self.osu_binaries.build_prefix, From 9dad0ce59c8009a3375e88d2224998b77f1566c0 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Sun, 31 Oct 2021 22:56:38 +0100 Subject: [PATCH 5/7] Add tutorial about fixtures --- .../img/fixtures-actual-deps-scoped.svg | 3 + docs/_static/img/fixtures-actual-deps.svg | 3 + docs/_static/img/fixtures-conceptual-deps.svg | 3 + docs/tutorial_fixtures.rst | 275 ++++++++++++++++++ docs/tutorial_tips_tricks.rst | 2 +- docs/tutorials.rst | 3 +- tutorials/fixtures/osu_benchmarks.py | 5 + 7 files changed, 292 insertions(+), 2 deletions(-) create mode 100644 docs/_static/img/fixtures-actual-deps-scoped.svg create mode 100644 docs/_static/img/fixtures-actual-deps.svg create mode 100644 docs/_static/img/fixtures-conceptual-deps.svg create mode 100644 docs/tutorial_fixtures.rst diff --git a/docs/_static/img/fixtures-actual-deps-scoped.svg b/docs/_static/img/fixtures-actual-deps-scoped.svg new file mode 100644 index 0000000000..4d491da566 --- /dev/null +++ b/docs/_static/img/fixtures-actual-deps-scoped.svg @@ -0,0 +1,3 @@ + + +
build_osu_benchmarks~daint:gpu+pgi@daint:gpu+pgi
build_osu_benchmarks~daint:gpu+pgi@daint:gpu+pgi
fetch_osu_benchmarks~daint@daint:gpu+pgi
fetch_osu_benchmarks~daint@daint:gpu+pgi
osu_bandwidth_test@daint:gpu+pgi
osu_bandwidth_test@daint:gpu+pgiViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/_static/img/fixtures-actual-deps.svg b/docs/_static/img/fixtures-actual-deps.svg new file mode 100644 index 0000000000..7436d94ba4 --- /dev/null +++ b/docs/_static/img/fixtures-actual-deps.svg @@ -0,0 +1,3 @@ + + +
osu_bandwidth_test@daint:gpu+gnu
osu_bandwidth_test@daint:gpu+gnu
build_osu_benchmarks~daint:gpu+gnu@daint:gpu+gnu
build_osu_benchmarks~daint:gpu+gnu@daint:gpu+gnu
build_osu_benchmarks~daint:gpu+pgi@daint:gpu+pgi
build_osu_benchmarks~daint:gpu+pgi@daint:gpu+pgi
build_osu_benchmarks~daint:gpu+intel@daint:gpu+intel
build_osu_benchmarks~daint:gpu+intel@daint:gpu+intel
fetch_osu_benchmarks~daint@daint:gpu+gnu
fetch_osu_benchmarks~daint@daint:gpu+gnu
osu_bandwidth_test@daint:gpu+pgi
osu_bandwidth_test@daint:gpu+pgi
osu_bandwidth_test@daint:gpu+intel
osu_bandwidth_test@daint:gpu+intelViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/_static/img/fixtures-conceptual-deps.svg b/docs/_static/img/fixtures-conceptual-deps.svg new file mode 100644 index 0000000000..98bf255d5d --- /dev/null +++ b/docs/_static/img/fixtures-conceptual-deps.svg @@ -0,0 +1,3 @@ + + +
osu_bandwidth_test
osu_bandwidth_test
build_osu_benchmarks~daint:gpu+gnu
build_osu_benchmarks~daint:gpu+gnu
build_osu_benchmarks~daint:gpu+pgi
build_osu_benchmarks~daint:gpu+pgi
build_osu_benchmarks~daint:gpu+intel
build_osu_benchmarks~daint:gpu+intel
fetch_osu_benchmarks~daint
fetch_osu_benchmarks~daintViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/docs/tutorial_fixtures.rst b/docs/tutorial_fixtures.rst new file mode 100644 index 0000000000..41bb4dfe96 --- /dev/null +++ b/docs/tutorial_fixtures.rst @@ -0,0 +1,275 @@ +Tutorial 4: Using Test Fixtures +=============================== + +.. versionadded:: 3.9.0 + +A fixture in ReFrame is a test that manages a resource of another test. +Fixtures can be chained to create essentially a graph of dependencies. +Similarly to test dependencies, the test that uses the fixture will not execute until its fixture has executed. +In this tutorial, we will rewrite the OSU benchmarks example presented in :doc:`tutorial_deps` using fixtures. +We will cover only the basic concepts of fixtures that will allow you to start using them in your tests. +For the full documentation of the test fixtures, you should refer to the :doc:`regression_test_api` documentation. + +The full example of the OSU benchmarks using test fixtures is shown below with the relevant parts highlighted: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 6- + :emphasize-lines: 6-43,53,67-68,85-86,106-107 + +Let's start from the leaf tests, i.e. the tests that execute the benchmarks (:class:`osu_latency_test`, :class:`osu_bandwidth_test` and :class:`osu_allreduce_test`). +As in the dependencies example, all these tests derive from the :class:`OSUBenchmarkTestBase`, where we define a fixture that will take care of generating the binaries of the tests: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 58 + +A test defines a fixture using the :func:`~reframe.core.pipeline.RegressionMixin.fixture` builtin and assigns it a name by assigning the return value of the builtin to a test variable, here ``osu_binaries``. +This name will be used later to access the resource managed by the fixture. + +As stated previously, a fixture is another full-fledged ReFrame test, here the :class:`build_osu_benchmarks` which will take care of building the OSU benchmarks. +Each fixture is associated with a scope. +This practically indicates at which level a fixture is shared with other tests. +There are four fixture scopes, which are listed below in decreasing order of generality: + +- ``session``: A fixture with this scope will be executed once per a ReFrame run session and will be shared across the whole run. +- ``partition``: A fixture with this scope will be executed once per partition and will be shared across all tests that run in that partition. +- ``environment``: A fixture with this scope will be executed once per partition and environment combination and will be shared across all tests that run with this partition and environment combination. +- ``test``: A fixture with this scope is private to the test and will be executed for each test case. + +In this example, we need to build once the OSU benchmarks for each partition and environment combination, so we use the ``environment`` scope. + +Accessing the fixture is very straightforward. +The fixture's result is accessible after the *setup* pipeline stage through the corresponding variable in the test that is defining it. +Since a fixture is a standard ReFrame test, you can access any information of the test. +The individual benchmarks do exactly that: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 69-76 + :emphasize-lines: 4-5 + +Here we construct the final executable path by accessing the standard :attr:`~reframe.core.pipeline.RegressionTest.stagedir` attribute of the test as well as the custom-defined :attr:`build_prefix` variable of the :class:`build_osu_benchmarks` fixture. + +Let's inspect now the :class:`build_osu_benchmarks` fixture: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 25-49 + :emphasize-lines: 5,9,12 + +It is obvious that it is a normal ReFrame test except that it does not need to be decorated with the :func:`@simple_test ` decorator. +This means that the test will only be executed if it is a fixture of another test. +If it was decorated, it would be executed both as a standalone test and as a fixture of another test. +Another detail is that this test does dot define the :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs` variables. +Fixtures inherit those variables from the test that owns them depending on the scope. + +Similarly to :class:`OSUBenchmarkTestBase`, this test uses a fixture that fetches the OSU benchmarks sources. +We could fetch the OSU benchmarks in this test, but we choose to separate the two primarily for demonstration purposes, but it would also make sense in cases that the data fetch is too slow. + +The ``osu_benchmarks`` fixture is defined at session scope, since we only need to download the benchmarks once for the whole session: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 58 + +The rest of the test is very straightforward. + +Let's inspect the last fixture, the :class:`fetch_osu_benchmarks`: + +.. literalinclude:: ../tutorials/fixtures/osu_benchmarks.py + :lines: 11-22 + :emphasize-lines: 8 + +There is nothing special to this test -- it is just an ordinary test -- except that we force it to execute locally by setting its :attr:`~reframe.core.pipeline.RegressionTest.local` variable. +The reason for that is that a fixture at session scope can execute with any partition/environment combination, so ReFrame could have to spawn a job in case it has chosen a remote partition to launch this fixture on. +For this reason, we simply force it execute locally regardless of the chosen partition. + +It is now time to run the new tests, but let us first list them: + +.. code-block:: bash + + export RFM_CONFIG_FILE=$(pwd)/tutorials/config/settings.py + reframe -c tutorials/fixtures/osu_benchmarks.py -l + +.. code-block:: console + + [ReFrame Setup] + version: 3.9.0 + command: 'reframe -c tutorials/fixtures/osu_benchmarks.py -l' + launched by: user@daint106 + working directory: '/users/user/Devel/reframe' + settings file: '/users/user/Devel/reframe/tutorials/config/settings.py' + check search path: '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py' + stage directory: '/users/user/Devel/reframe/stage' + output directory: '/users/user/Devel/reframe/output' + + [List of matched checks] + - osu_latency_test (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + - osu_allreduce_test_8 (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + - osu_allreduce_test_2 (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + - osu_allreduce_test_4 (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + - osu_bandwidth_test (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + - osu_allreduce_test_16 (found in '/users/user/Devel/reframe/tutorials/fixtures/osu_benchmarks.py') + Found 6 check(s) + + Log file(s) saved in '/tmp/rfm-dlkc1vb_.log' + +Notice that only the leaf tests are listed and not their fixtures. +Listing the tests in detailed mode, however, using the :option:`-L` option, you will see all the generated fixtures: + + +.. code-block:: bash + + reframe -c tutorials/fixtures/osu_benchmarks.py -n osu_bandwidth_test -L + +ReFrame will generate 4 fixtures for this test based on the partition and environment configurations for the current system. +The following figure shows the generated fixtures as well as their conceptual dependencies. + +.. figure:: _static/img/fixtures-conceptual-deps.svg + :align: center + + :sub:`Expanded fixtures and dependencies for the OSU benchmarks example.` + +Notice how the :class:`build_osu_benchmarks` fixture is populated three times, for each partition and environment combination, and the :class:`fetch_osu_benchmarks` is generated only once. +Tests in a single ReFrame session must have unique names, so the fixture class name is mangled by the framework to generate a unique name in the test dependency DAG. +A *scope* part is added to the base name of the fixture, which in this figure is indicated with red color. + +Under the hood, fixtures use the test dependency mechanism which is described in :doc:`dependencies`. +The dependencies shown in the previous figure are conceptual. +A single test in ReFrame generates a series of test cases for all the combinations of valid systems and valid programming environments and the actual dependencies are expressed in this more fine-grained layer, which is also the layer at which the execution of tests is scheduled. + +The following figure shows how the above graph translates into the actual DAG of test cases. + +.. figure:: _static/img/fixtures-actual-deps.svg + :align: center + + :sub:`The actual dependencies for the OSU benchmarks example using fixtures.` + + +The first thing to notice here is how the individual test cases of :class:`osu_bandwidth_test` depend only the specific fixtures for their scope: +when :class:`osu_bandwidth_test` runs on the ``daint:gpu`` partition using the ``gnu`` compiler it will only depend on the :class:`build_osu_benchmarks~daint:gpu+gnu` fixture. +The second thing to notice is where the :class:`fetch_osu_benchmarks~daint` fixture will run. +Since this is a *session* fixture, ReFrame has arbitrarily chosen to run it on ``daint:gpu`` using the ``gnu`` environment. +A session fixture can run on any combination of valid partitions and environments. +The following figure shows how the test dependency DAG is concretized when we scope the valid programming environments from the command line using ``-p pgi``. + + +.. figure:: _static/img/fixtures-actual-deps-scoped.svg + :align: center + + :sub:`The dependency graph concretized for the 'pgi' environment only.` + + +Notice how the :class:`fetch_osu_benchmarks~daint` fixture is selected to run in the only valid partition/environment combination. + +The following listing shows the output of running the tutorial examples. + +.. code-block:: console + + [==========] Running 10 check(s) + [==========] Started on Sun Oct 31 22:00:28 2021 + + [----------] started processing fetch_osu_benchmarks~daint (Fetch OSU benchmarks) + [ RUN ] fetch_osu_benchmarks~daint on daint:gpu using gnu + [----------] finished processing fetch_osu_benchmarks~daint (Fetch OSU benchmarks) + + [----------] started processing build_osu_benchmarks~daint:gpu+gnu (Build OSU benchmarks) + [ RUN ] build_osu_benchmarks~daint:gpu+gnu on daint:gpu using gnu + [ DEP ] build_osu_benchmarks~daint:gpu+gnu on daint:gpu using gnu + [----------] finished processing build_osu_benchmarks~daint:gpu+gnu (Build OSU benchmarks) + + [----------] started processing build_osu_benchmarks~daint:gpu+intel (Build OSU benchmarks) + [ RUN ] build_osu_benchmarks~daint:gpu+intel on daint:gpu using intel + [ DEP ] build_osu_benchmarks~daint:gpu+intel on daint:gpu using intel + [----------] finished processing build_osu_benchmarks~daint:gpu+intel (Build OSU benchmarks) + + [----------] started processing build_osu_benchmarks~daint:gpu+pgi (Build OSU benchmarks) + [ RUN ] build_osu_benchmarks~daint:gpu+pgi on daint:gpu using pgi + [ DEP ] build_osu_benchmarks~daint:gpu+pgi on daint:gpu using pgi + [----------] finished processing build_osu_benchmarks~daint:gpu+pgi (Build OSU benchmarks) + + [----------] started processing osu_allreduce_test_16 (OSU Allreduce test) + [ RUN ] osu_allreduce_test_16 on daint:gpu using gnu + [ DEP ] osu_allreduce_test_16 on daint:gpu using gnu + [ RUN ] osu_allreduce_test_16 on daint:gpu using intel + [ DEP ] osu_allreduce_test_16 on daint:gpu using intel + [ RUN ] osu_allreduce_test_16 on daint:gpu using pgi + [ DEP ] osu_allreduce_test_16 on daint:gpu using pgi + [----------] finished processing osu_allreduce_test_16 (OSU Allreduce test) + + [----------] started processing osu_allreduce_test_8 (OSU Allreduce test) + [ RUN ] osu_allreduce_test_8 on daint:gpu using gnu + [ DEP ] osu_allreduce_test_8 on daint:gpu using gnu + [ RUN ] osu_allreduce_test_8 on daint:gpu using intel + [ DEP ] osu_allreduce_test_8 on daint:gpu using intel + [ RUN ] osu_allreduce_test_8 on daint:gpu using pgi + [ DEP ] osu_allreduce_test_8 on daint:gpu using pgi + [----------] finished processing osu_allreduce_test_8 (OSU Allreduce test) + + [----------] started processing osu_allreduce_test_4 (OSU Allreduce test) + [ RUN ] osu_allreduce_test_4 on daint:gpu using gnu + [ DEP ] osu_allreduce_test_4 on daint:gpu using gnu + [ RUN ] osu_allreduce_test_4 on daint:gpu using intel + [ DEP ] osu_allreduce_test_4 on daint:gpu using intel + [ RUN ] osu_allreduce_test_4 on daint:gpu using pgi + [ DEP ] osu_allreduce_test_4 on daint:gpu using pgi + [----------] finished processing osu_allreduce_test_4 (OSU Allreduce test) + + [----------] started processing osu_allreduce_test_2 (OSU Allreduce test) + [ RUN ] osu_allreduce_test_2 on daint:gpu using gnu + [ DEP ] osu_allreduce_test_2 on daint:gpu using gnu + [ RUN ] osu_allreduce_test_2 on daint:gpu using intel + [ DEP ] osu_allreduce_test_2 on daint:gpu using intel + [ RUN ] osu_allreduce_test_2 on daint:gpu using pgi + [ DEP ] osu_allreduce_test_2 on daint:gpu using pgi + [----------] finished processing osu_allreduce_test_2 (OSU Allreduce test) + + [----------] started processing osu_bandwidth_test (OSU bandwidth test) + [ RUN ] osu_bandwidth_test on daint:gpu using gnu + [ DEP ] osu_bandwidth_test on daint:gpu using gnu + [ RUN ] osu_bandwidth_test on daint:gpu using intel + [ DEP ] osu_bandwidth_test on daint:gpu using intel + [ RUN ] osu_bandwidth_test on daint:gpu using pgi + [ DEP ] osu_bandwidth_test on daint:gpu using pgi + [----------] finished processing osu_bandwidth_test (OSU bandwidth test) + + [----------] started processing osu_latency_test (OSU latency test) + [ RUN ] osu_latency_test on daint:gpu using gnu + [ DEP ] osu_latency_test on daint:gpu using gnu + [ RUN ] osu_latency_test on daint:gpu using intel + [ DEP ] osu_latency_test on daint:gpu using intel + [ RUN ] osu_latency_test on daint:gpu using pgi + [ DEP ] osu_latency_test on daint:gpu using pgi + [----------] finished processing osu_latency_test (OSU latency test) + + [----------] waiting for spawned checks to finish + [ OK ] ( 1/22) fetch_osu_benchmarks~daint on daint:gpu using gnu [compile: 0.009s run: 2.761s total: 2.802s] + [ OK ] ( 2/22) build_osu_benchmarks~daint:gpu+gnu on daint:gpu using gnu [compile: 25.758s run: 0.056s total: 104.626s] + [ OK ] ( 3/22) build_osu_benchmarks~daint:gpu+pgi on daint:gpu using pgi [compile: 33.936s run: 70.452s total: 104.473s] + [ OK ] ( 4/22) build_osu_benchmarks~daint:gpu+intel on daint:gpu using intel [compile: 44.565s run: 65.010s total: 143.664s] + [ OK ] ( 5/22) osu_allreduce_test_4 on daint:gpu using gnu [compile: 0.011s run: 78.717s total: 101.428s] + [ OK ] ( 6/22) osu_allreduce_test_2 on daint:gpu using pgi [compile: 0.014s run: 88.060s total: 101.409s] + [ OK ] ( 7/22) osu_latency_test on daint:gpu using pgi [compile: 0.009s run: 101.325s total: 101.375s] + [ OK ] ( 8/22) osu_allreduce_test_8 on daint:gpu using pgi [compile: 0.013s run: 76.031s total: 102.005s] + [ OK ] ( 9/22) osu_allreduce_test_2 on daint:gpu using gnu [compile: 0.011s run: 85.525s total: 101.974s] + [ OK ] (10/22) osu_allreduce_test_4 on daint:gpu using pgi [compile: 0.011s run: 82.847s total: 102.407s] + [ OK ] (11/22) osu_allreduce_test_8 on daint:gpu using gnu [compile: 0.010s run: 77.818s total: 106.993s] + [ OK ] (12/22) osu_latency_test on daint:gpu using gnu [compile: 0.012s run: 103.641s total: 106.858s] + [ OK ] (13/22) osu_bandwidth_test on daint:gpu using pgi [compile: 0.011s run: 157.129s total: 164.087s] + [ OK ] (14/22) osu_bandwidth_test on daint:gpu using gnu [compile: 0.010s run: 154.343s total: 164.540s] + [ OK ] (15/22) osu_allreduce_test_8 on daint:gpu using intel [compile: 0.010s run: 194.643s total: 207.980s] + [ OK ] (16/22) osu_allreduce_test_2 on daint:gpu using intel [compile: 0.013s run: 201.145s total: 207.983s] + [ OK ] (17/22) osu_allreduce_test_4 on daint:gpu using intel [compile: 0.016s run: 198.143s total: 208.335s] + [ OK ] (18/22) osu_latency_test on daint:gpu using intel [compile: 0.010s run: 208.271s total: 208.312s] + [ OK ] (19/22) osu_allreduce_test_16 on daint:gpu using pgi [compile: 0.013s run: 215.854s total: 248.101s] + [ OK ] (20/22) osu_allreduce_test_16 on daint:gpu using gnu [compile: 0.010s run: 213.190s total: 248.731s] + [ OK ] (21/22) osu_allreduce_test_16 on daint:gpu using intel [compile: 0.010s run: 194.339s total: 210.962s] + [ OK ] (22/22) osu_bandwidth_test on daint:gpu using intel [compile: 0.022s run: 267.171s total: 270.475s] + [----------] all spawned checks have finished + + [ PASSED ] Ran 22/22 test case(s) from 10 check(s) (0 failure(s), 0 skipped) + [==========] Finished on Sun Oct 31 22:07:25 2021 + Run report saved in '/users/user/.reframe/reports/run-report.json' + Log file(s) saved in '/tmp/rfm-qst7lvou.log' + + +.. tip:: + A reasonable question that arises is when should I use fixtures and when dependencies? + + The rule of thumb is use fixtures if your test needs to use any resource of the target test and use dependencies if you simply want to impose an order of execution for your tests. diff --git a/docs/tutorial_tips_tricks.rst b/docs/tutorial_tips_tricks.rst index bc0ca30c47..304a87bf5e 100644 --- a/docs/tutorial_tips_tricks.rst +++ b/docs/tutorial_tips_tricks.rst @@ -1,5 +1,5 @@ =========================== -Tutorial 4: Tips and Tricks +Tutorial 6: Tips and Tricks =========================== .. versionadded:: 3.4 diff --git a/docs/tutorials.rst b/docs/tutorials.rst index c188e1e2af..83b215a4ec 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -8,8 +8,9 @@ ReFrame Tutorials tutorial_basics tutorial_advanced tutorial_deps - tutorial_tips_tricks + tutorial_fixtures tutorial_build_automation + tutorial_tips_tricks Online Tutorials diff --git a/tutorials/fixtures/osu_benchmarks.py b/tutorials/fixtures/osu_benchmarks.py index ff8a56d15f..492a5e3bea 100644 --- a/tutorials/fixtures/osu_benchmarks.py +++ b/tutorials/fixtures/osu_benchmarks.py @@ -1,3 +1,8 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + import os import reframe as rfm import reframe.utility.sanity as sn From 64c42cfd7577485d54c4377fe0155ad7ab9308ae Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 2 Nov 2021 10:20:32 +0100 Subject: [PATCH 6/7] Update docs --- docs/tutorial_fixtures.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial_fixtures.rst b/docs/tutorial_fixtures.rst index 41bb4dfe96..7286337838 100644 --- a/docs/tutorial_fixtures.rst +++ b/docs/tutorial_fixtures.rst @@ -270,6 +270,6 @@ The following listing shows the output of running the tutorial examples. .. tip:: - A reasonable question that arises is when should I use fixtures and when dependencies? + A reasonable question is how to choose between fixtures and dependencies? The rule of thumb is use fixtures if your test needs to use any resource of the target test and use dependencies if you simply want to impose an order of execution for your tests. From a2e18c10b6c8d72b795ac54989b765c7bb156270 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Thu, 4 Nov 2021 22:28:29 +0100 Subject: [PATCH 7/7] Address PR comments --- docs/tutorial_fixtures.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/tutorial_fixtures.rst b/docs/tutorial_fixtures.rst index 7286337838..b900ce2ff5 100644 --- a/docs/tutorial_fixtures.rst +++ b/docs/tutorial_fixtures.rst @@ -30,7 +30,7 @@ Each fixture is associated with a scope. This practically indicates at which level a fixture is shared with other tests. There are four fixture scopes, which are listed below in decreasing order of generality: -- ``session``: A fixture with this scope will be executed once per a ReFrame run session and will be shared across the whole run. +- ``session``: A fixture with this scope will be executed once per ReFrame run session and will be shared across the whole run. - ``partition``: A fixture with this scope will be executed once per partition and will be shared across all tests that run in that partition. - ``environment``: A fixture with this scope will be executed once per partition and environment combination and will be shared across all tests that run with this partition and environment combination. - ``test``: A fixture with this scope is private to the test and will be executed for each test case. @@ -57,7 +57,7 @@ Let's inspect now the :class:`build_osu_benchmarks` fixture: It is obvious that it is a normal ReFrame test except that it does not need to be decorated with the :func:`@simple_test ` decorator. This means that the test will only be executed if it is a fixture of another test. If it was decorated, it would be executed both as a standalone test and as a fixture of another test. -Another detail is that this test does dot define the :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs` variables. +Another detail is that this test does not define the :attr:`~reframe.core.pipeline.RegressionTest.valid_systems` and :attr:`~reframe.core.pipeline.RegressionTest.valid_prog_environs` variables. Fixtures inherit those variables from the test that owns them depending on the scope. Similarly to :class:`OSUBenchmarkTestBase`, this test uses a fixture that fetches the OSU benchmarks sources. @@ -78,7 +78,7 @@ Let's inspect the last fixture, the :class:`fetch_osu_benchmarks`: There is nothing special to this test -- it is just an ordinary test -- except that we force it to execute locally by setting its :attr:`~reframe.core.pipeline.RegressionTest.local` variable. The reason for that is that a fixture at session scope can execute with any partition/environment combination, so ReFrame could have to spawn a job in case it has chosen a remote partition to launch this fixture on. -For this reason, we simply force it execute locally regardless of the chosen partition. +For this reason, we simply force it to execute locally regardless of the chosen partition. It is now time to run the new tests, but let us first list them: @@ -126,7 +126,7 @@ The following figure shows the generated fixtures as well as their conceptual de :sub:`Expanded fixtures and dependencies for the OSU benchmarks example.` -Notice how the :class:`build_osu_benchmarks` fixture is populated three times, for each partition and environment combination, and the :class:`fetch_osu_benchmarks` is generated only once. +Notice how the :class:`build_osu_benchmarks` fixture is populated three times, once for each partition and environment combination, and the :class:`fetch_osu_benchmarks` is generated only once. Tests in a single ReFrame session must have unique names, so the fixture class name is mangled by the framework to generate a unique name in the test dependency DAG. A *scope* part is added to the base name of the fixture, which in this figure is indicated with red color.