diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 8ba19b7d0d..898f93cf33 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -41,8 +41,8 @@ checked_exec() run_tutorial_checks() { - cmd="./bin/reframe -C tutorials/config/settings.py -J account=jenscscs \ ---save-log-files --flex-alloc-nodes=2 -r -x HelloThreadedExtendedTest $@" + cmd="./bin/reframe -vv -C tutorials/config/settings.py -J account=jenscscs \ +--save-log-files --flex-alloc-nodes=2 -r -x HelloThreadedExtendedTest|BZip2.*Check $@" echo "[INFO] Running tutorial checks with \`$cmd'" checked_exec $cmd } diff --git a/docs/tutorial_build_automation.rst b/docs/tutorial_build_automation.rst new file mode 100644 index 0000000000..85284ffd69 --- /dev/null +++ b/docs/tutorial_build_automation.rst @@ -0,0 +1,183 @@ +========================================================== +Tutorial 5: Using Build Automation Tools As a Build System +========================================================== + +In this tutorial we will present how to use `Easybuild `__ and `Spack `__ as a build system for a ReFrame test. +The example uses the configuration file presented in :doc:`tutorial_basics`, which you can find in ``tutorials/config/settings.py``. +We also assume that the reader is already familiar with the concepts presented in the basic tutorial and has a working knowledge of EasyBuild and Spack. +Finally, to avoid specifying the tutorial configuration file each time you run the test, make sure to export it here: + +.. code:: bash + + export RFM_CONFIG_FILE=$(pwd)/tutorials/config/mysettings.py + + + +Using EasyBuild to Build the Test Code +-------------------------------------- + +.. versionadded:: 3.5.0 + + +Let's consider a simple ReFrame test that installs ``bzip2-1.0.6`` given the easyconfig `bzip2-1.0.6.eb `__ and checks that the installed version is correct. +The following code block shows the check, highlighting the lines specific to this tutorial: + +.. literalinclude:: ../tutorials/build_systems/easybuild/eb_test.py + :lines: 6- + :emphasize-lines: 12,14-17,19-21 + +The test looks pretty standard except for the highlighted blocks. +Let's have a look first to the block in the :class:`BZip2Check` class. + +The first thing is to specify that the EasyBuild build system will be used. +This is done by setting :attr:`~reframe.core.pipeline.RegressionTest.build_system` to ``'EasyBuild'``. +Then, the software to be installed is passed as a list to :attr:`~reframe.core.buildsystems.EasyBuild.easyconfigs`. +Here only one easyconfig is given, but more than one can be passed. +Finally, through :attr:`~reframe.core.buildsystems.EasyBuild.options`, command line options can be passed to the ``eb`` executable. +In this test we pass ``-f`` to make sure that ``bzip2`` will be built even if the module already exists externally. + +For this test, ReFrame generates the following command to build and install the easyconfig: + +.. code-block:: console + + export EASYBUILD_BUILDPATH={stagedir}/easybuild/build + export EASYBUILD_INSTALLPATH={stagedir}/easybuild + export EASYBUILD_PREFIX={stagedir}/easybuild + export EASYBUILD_SOURCEPATH={stagedir}/easybuild + eb bzip2-1.0.6.eb -f + +ReFrame will keep all the files generated by EasyBuild (sources, temporary files, installed software and the corresponding modules) under the test's stage directory. +For this reason it sets the relevant EasyBuild environment variables. + +.. tip:: + + Users may set the EasyBuild prefix to a different location by setting the :attr:`~reframe.core.buildsystems.EasyBuild.prefix` attribute of the build system. + This allows you to have the built software installed upon successful completion of the build phase, but if the test fails in a later stage (sanity, performance), the installed software will not be cleaned up automatically. + +.. note:: + + ReFrame assumes that the ``eb`` executable is available on the system where the compilation is run (typically the local host where ReFrame is executed). + + +Now that we know everything related to building and installing the code, we can move to the part dealing with running it. +To run the code, the generated modules need to be loaded in order to make the software available. +The modules can be accessed through :attr:`~reframe.core.buildsystems.Easybuild.generated_modules`, however, they are available only after EasyBuild completes the installation. +This means that :attr:`~reframe.core.pipeline.RegressionTest.modules` can be set only after the build phase finishes. +For that, we can set :attr:`~reframe.core.pipeline.RegressionTest.modules` in a class method wrapped by the :py:func:`~reframe.core.RegressionTest.run_before` built-in, specifying the ``run`` phase. +This test will then run the following commands: + +.. code-block:: console + + module load bzip/1.0.6 + bzip2 --help + + +-------------------------- +Packaging the installation +-------------------------- + +The EasyBuild build system offers a way of packaging the installation via EasyBuild's packaging support. +To use this feature, `the FPM package manager `__ must be available. +By setting the dictionary :attr:`~reframe.core.buildsystems.Easybuild.package_opts` in the test, ReFrame will pass ``--package-{key}={val}`` to the EasyBuild invocation. +For instance, the following can be set to package the installations as an rpm file: + +.. code-block:: python + + self.keep_files = ['easybuild/packages'] + self.build_system.package_opts = { + 'type': 'rpm', + } + +The packages are generated by EasyBuild in the stage directory. +To retain them after the test succeeds, :attr:`~reframe.core.pipeline.RegressionTest.keep_files` needs to be set. + + + +Using Spack to Build the Test Code +---------------------------------- + + +.. versionadded:: 3.6.1 + + +This example is the equivalent to the previous one, except that it uses Spack to build ``bzip2``. +Here is the test's code: + +.. literalinclude:: ../tutorials/build_systems/spack/spack_test.py + :lines: 6- + :emphasize-lines: 12,14-16 + + +When :attr:`~reframe.core.pipeline.RegressionTest.build_system` is set to ``'Spack'``, ReFrame will leverage Spack environments in order to build the test code. +For this reason, currently, users must specify an environment. +ReFrame treats Spack environments as *test resources* so it expects to find them under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. +Here is the directory structure for the test in this particular example that we show here: + +.. code:: console + + tutorials/build_systems/spack/ + ├── spack_test.py + └── src + └── myenv + └── spack.yaml + + +We could have placed ``spack.yaml`` directly under the ``src/`` directory, in which case we would need to specify ``'.'`` as an environment. +For reference, here are the contents of ``spack.yaml``: + +.. literalinclude:: ../tutorials/build_systems/spack/src/myenv/spack.yaml + + +As with every other test, ReFrame will copy the test's resources to its stage directory before building it. +ReFrame will then activate the environment and install the associated specs as in this case. +Optionally, we can add more specs to the environment by setting the :attr:`~reframe.core.buildsystems.Spack.specs` attribute of the build system. +Here is what ReFrame generates as a build script in this example: + +.. code:: bash + + . $SPACK_ROOT/share/spack/setup-env.sh + spack env activate -V -d myenv + spack install + +Any additional specs specified inside the ReFrame test will be added using the ``spack add`` command. +As you might have noticed ReFrame expects that Spack is already installed on the system. +The packages specified in the environment and the tests will be installed in the test's stage directory, where the environment is copied before building. +Here is the stage directory structure: + +.. code:: console + + stage/generic/default/builtin/BZip2SpackCheck/ + ├── myenv + │   ├── spack + │   │   ├── opt + │   │   │   └── spack + │   │   │   ├── bin + │   │   │   └── darwin-catalina-skylake + │   │   └── share + │   │   └── spack + │   │   └── modules + │   ├── spack.lock + │   └── spack.yaml + ├── rfm_BZip2SpackCheck_build.err + ├── rfm_BZip2SpackCheck_build.out + ├── rfm_BZip2SpackCheck_build.sh + ├── rfm_BZip2SpackCheck_job.err + ├── rfm_BZip2SpackCheck_job.out + └── rfm_BZip2SpackCheck_job.sh + + +Finally, here is the generated run script that ReFrame uses to run the test, once its build has succeeded: + +.. code-block:: bash + + #!/bin/bash + . $SPACK_ROOT/share/spack/setup-env.sh + spack env activate -V -d myenv + bzip2 --help + +From this point on, sanity and performance checking are exactly identical to any other ReFrame test. + +.. tip:: + + While developing a test using Spack or EasyBuild as a build system, it can be useful to run ReFrame with the :option:`--keep-stage-files` and :option:`--dont-restage` options to prevent ReFrame from removing the test's stage directory upon successful completion of the test. + For this particular type of test, these options will avoid having to rebuild the required package dependencies every time the test is retried. diff --git a/docs/tutorials.rst b/docs/tutorials.rst index 174bae3c2f..c188e1e2af 100644 --- a/docs/tutorials.rst +++ b/docs/tutorials.rst @@ -9,6 +9,7 @@ ReFrame Tutorials tutorial_advanced tutorial_deps tutorial_tips_tricks + tutorial_build_automation Online Tutorials diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 3287d885db..d1fa3102ca 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -703,7 +703,7 @@ class EasyBuild(BuildSystem): ReFrame will use EasyBuild to build and install the code in the test's stage directory by default. ReFrame uses environment variables to - configure EasyBuild for running, so Users can pass additional options to + configure EasyBuild for running, so users can pass additional options to the ``eb`` command and modify the default behaviour. .. versionadded:: 3.5.0 @@ -832,7 +832,8 @@ class Spack(BuildSystem): #: :default: :class:`None` environment = fields.TypedField(typ.Str[r'\S+'], _UndefinedType) - #: The list of specs to build and install within the given environment. + #: A list of additional specs to build and install within the given + #: environment. #: #: ReFrame will add the specs to the active environment by emititing the #: following command: diff --git a/reframe/frontend/cli.py b/reframe/frontend/cli.py index bb2e112b95..ba6a7c8b1f 100644 --- a/reframe/frontend/cli.py +++ b/reframe/frontend/cli.py @@ -857,8 +857,6 @@ def _case_failed(t): printer.debug(dependencies.format_deps(testgraph)) if options.restore_session is not None: testgraph, restored_cases = report.restore_dangling(testgraph) - print(dependencies.format_deps(testgraph)) - print(restored_cases) testcases = dependencies.toposort( testgraph, diff --git a/tutorials/build_systems/easybuild/eb_test.py b/tutorials/build_systems/easybuild/eb_test.py new file mode 100644 index 0000000000..17d6bb2ae8 --- /dev/null +++ b/tutorials/build_systems/easybuild/eb_test.py @@ -0,0 +1,30 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class BZip2EBCheck(rfm.RegressionTest): + descr = 'Demo test using EasyBuild to build the test code' + valid_systems = ['*'] + valid_prog_environs = ['builtin'] + executable = 'bzip2' + executable_opts = ['--help'] + build_system = 'EasyBuild' + + @run_before('compile') + def setup_build_system(self): + self.build_system.easyconfigs = ['bzip2-1.0.6.eb'] + self.build_system.options = ['-f'] + + @run_before('run') + def prepare_run(self): + self.modules = self.build_system.generated_modules + + @sanity_function + def assert_version(self): + return sn.assert_found(r'Version 1.0.6', self.stderr) diff --git a/tutorials/build_systems/spack/spack_test.py b/tutorials/build_systems/spack/spack_test.py new file mode 100644 index 0000000000..9c7fe91a22 --- /dev/null +++ b/tutorials/build_systems/spack/spack_test.py @@ -0,0 +1,25 @@ +# Copyright 2016-2021 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +import reframe as rfm +import reframe.utility.sanity as sn + + +@rfm.simple_test +class BZip2SpackCheck(rfm.RegressionTest): + descr = 'Demo test using Spack to build the test code' + valid_systems = ['*'] + valid_prog_environs = ['builtin'] + executable = 'bzip2' + executable_opts = ['--help'] + build_system = 'Spack' + + @run_before('compile') + def setup_build_system(self): + self.build_system.environment = 'myenv' + + @sanity_function + def assert_version(self): + return sn.assert_found(r'Version 1.0.6', self.stderr) diff --git a/tutorials/build_systems/spack/src/myenv/spack.yaml b/tutorials/build_systems/spack/src/myenv/spack.yaml new file mode 100644 index 0000000000..61273fdde6 --- /dev/null +++ b/tutorials/build_systems/spack/src/myenv/spack.yaml @@ -0,0 +1,9 @@ +spack: + specs: + - bzip2@1.0.6 + concretization: together + config: + install_tree: spack/opt/spack + module_roots: + tcl: spack/share/spack/modules + lmod: spack/share/spack/lmod