From aa31af996eee4357a481b4fe27306796da4bccb3 Mon Sep 17 00:00:00 2001 From: Rafael Alejandro Sarmiento Perez Date: Wed, 28 Jul 2021 18:22:56 +0300 Subject: [PATCH 1/5] add dockerfile and config entry --- ci-scripts/dockerfiles/tutorials.dockerfile | 36 +++++++++++++++++++++ tutorials/config/settings.py | 14 ++++++++ 2 files changed, 50 insertions(+) create mode 100644 ci-scripts/dockerfiles/tutorials.dockerfile diff --git a/ci-scripts/dockerfiles/tutorials.dockerfile b/ci-scripts/dockerfiles/tutorials.dockerfile new file mode 100644 index 0000000000..50b8a9c0eb --- /dev/null +++ b/ci-scripts/dockerfiles/tutorials.dockerfile @@ -0,0 +1,36 @@ +# +# Execute this from the top-level ReFrame source directory +# + + +FROM reframehpc/rfm-ci-base:lmod + +ENV _SPACK_VER=0.16 +ENV _EB_VER=4.4.1 + +# ReFrame user +RUN useradd -ms /bin/bash rfmuser + +USER rfmuser + +# Install Spack +RUN git clone https://github.com/spack/spack ~/spack && \ + cd ~/spack && \ + git checkout releases/v${_SPACK_VER} + +RUN pip3 install easybuild==${_EB_VER} + +ENV PATH="/home/rfmuser/.local/bin:${PATH}" + +# Install ReFrame from the current directory +COPY --chown=rfmuser . /home/rfmuser/reframe/ + +WORKDIR /home/rfmuser/reframe + +RUN ./bootstrap.sh + +RUN echo '. /usr/local/lmod/lmod/init/profile && . /home/rfmuser/spack/share/spack/setup-env.sh' > /home/rfmuser/setup.sh + +ENV BASH_ENV /home/rfmuser/setup.sh + +CMD ["/bin/bash", "-c", "./bin/reframe -r -C tutorials/config/settings.py -R -c tutorials/build_systems --system tutorials-docker"] diff --git a/tutorials/config/settings.py b/tutorials/config/settings.py index 30dc4d2509..323e922d4c 100644 --- a/tutorials/config/settings.py +++ b/tutorials/config/settings.py @@ -23,6 +23,20 @@ } ] }, + { + 'name': 'tutorials-docker', + 'descr': 'Container for running the build system tutorials', + 'hostnames': ['docker'], + 'modules_system': 'lmod', + 'partitions': [ + { + 'name': 'default', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['builtin'], + } + ] + }, { 'name': 'daint', 'descr': 'Piz Daint Supercomputer', From 878d37eb165f33f540728182a9232dd1fc7eb004 Mon Sep 17 00:00:00 2001 From: Rafael Alejandro Sarmiento Perez Date: Tue, 17 Aug 2021 11:13:49 +0200 Subject: [PATCH 2/5] fixes for the spack check --- ci-scripts/dockerfiles/tutorials.dockerfile | 4 ++++ tutorials/build_systems/spack/spack_test.py | 1 + 2 files changed, 5 insertions(+) diff --git a/ci-scripts/dockerfiles/tutorials.dockerfile b/ci-scripts/dockerfiles/tutorials.dockerfile index 50b8a9c0eb..8802b9b1ce 100644 --- a/ci-scripts/dockerfiles/tutorials.dockerfile +++ b/ci-scripts/dockerfiles/tutorials.dockerfile @@ -8,6 +8,10 @@ FROM reframehpc/rfm-ci-base:lmod ENV _SPACK_VER=0.16 ENV _EB_VER=4.4.1 +# Required utilities +RUN apt-get -y update && \ + apt-get -y install curl + # ReFrame user RUN useradd -ms /bin/bash rfmuser diff --git a/tutorials/build_systems/spack/spack_test.py b/tutorials/build_systems/spack/spack_test.py index 9c7fe91a22..98a95d45ef 100644 --- a/tutorials/build_systems/spack/spack_test.py +++ b/tutorials/build_systems/spack/spack_test.py @@ -19,6 +19,7 @@ class BZip2SpackCheck(rfm.RegressionTest): @run_before('compile') def setup_build_system(self): self.build_system.environment = 'myenv' + self.build_system.specs = ['bzip2@1.0.6'] @sanity_function def assert_version(self): From c13e2446c63fcf6ef0d681fb58fd82123d642e98 Mon Sep 17 00:00:00 2001 From: Theofilos Manitaras Date: Wed, 18 Aug 2021 14:23:14 +0200 Subject: [PATCH 3/5] Add the tutorial testing to the github actions --- .github/workflows/main.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0fd0eea30f..6e6fed2a07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,6 +47,17 @@ jobs: run: | docker run reframe:${{ matrix.modules-version }} + tutorialtest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Build Image for Tutorial Tests + run: | + docker build -f ci-scripts/dockerfiles/tutorials.dockerfile -t reframe:tutorials . + - name: Run Tutorial Tests + run: | + docker run reframe:tutorials + unusedimports: runs-on: ubuntu-latest steps: From 152c8d7e55c9a08e54a092b2d8dc92e7e5b65f1a Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Fri, 3 Sep 2021 10:16:40 +0200 Subject: [PATCH 4/5] using automatic spack env creation --- docs/tutorial_build_automation.rst | 44 +++++++------------ tutorials/build_systems/spack/spack_test.py | 1 - .../build_systems/spack/src/myenv/spack.yaml | 9 ---- 3 files changed, 15 insertions(+), 39 deletions(-) delete mode 100644 tutorials/build_systems/spack/src/myenv/spack.yaml diff --git a/docs/tutorial_build_automation.rst b/docs/tutorial_build_automation.rst index 9aab28eb3b..6fa57c3077 100644 --- a/docs/tutorial_build_automation.rst +++ b/docs/tutorial_build_automation.rst @@ -110,36 +110,23 @@ Here is the test's code: When :attr:`~reframe.core.pipeline.RegressionTest.build_system` is set to ``'Spack'``, ReFrame will leverage Spack environments in order to build the test code. If the environment is not specified by the user, ReFrame will automatically create one inside the stage directory. -ReFrame treats Spack environments specified by the user as *test resources* so it expects to find them under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. -Here is the directory structure for the test in this particular example that we show here: - -.. code:: console - - tutorials/build_systems/spack/ - ├── spack_test.py - └── src - └── myenv - └── spack.yaml - - -We could have placed ``spack.yaml`` directly under the ``src/`` directory, in which case we would need to specify ``'.'`` as an environment. -For reference, here are the contents of ``spack.yaml``: - -.. literalinclude:: ../tutorials/build_systems/spack/src/myenv/spack.yaml +That's the case in this example, where the software to be built is specified through the attribute :attr:`~reframe.core.buildsystems.Spack.specs` +When Spack environments are specified by the user, ReFrame treats them as *test resources* so it expects to find them under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. As with every other test, ReFrame will copy the test's resources to its stage directory before building it. -ReFrame will then activate the environment and install the associated specs as in this case. -Optionally, we can add more specs to the environment by setting the :attr:`~reframe.core.buildsystems.Spack.specs` attribute of the build system. +ReFrame will then activate the generated environment, add the given specs using the ``spack add`` command and install them. Here is what ReFrame generates as a build script in this example: .. code:: bash . "$(spack location --spack-root)/share/spack/setup-env.sh" - spack env activate -V -d myenv + spack env create -d rfm_spack_env + spack env activate -V -d rfm_spack_env + spack config add "config:install_tree:root:opt/spack" + spack add bzip2@1.0.6 spack install -Any additional specs specified inside the ReFrame test will be added using the ``spack add`` command. As you might have noticed ReFrame expects that Spack is already installed on the system. The packages specified in the environment and the tests will be installed in the test's stage directory, where the environment is copied before building. Here is the stage directory structure: @@ -147,15 +134,12 @@ Here is the stage directory structure: .. code:: console stage/generic/default/builtin/BZip2SpackCheck/ - ├── myenv + ├── rfm_spack_env │   ├── spack - │   │   ├── opt - │   │   │   └── spack - │   │   │   ├── bin - │   │   │   └── darwin-catalina-skylake - │   │   └── share - │   │   └── spack - │   │   └── modules + │   │   └── opt + │   │      └── spack + │   │      ├── bin + │   │      └── darwin-catalina-skylake │   ├── spack.lock │   └── spack.yaml ├── rfm_BZip2SpackCheck_build.err @@ -172,7 +156,9 @@ Finally, here is the generated run script that ReFrame uses to run the test, onc #!/bin/bash . "$(spack location --spack-root)/share/spack/setup-env.sh" - spack env activate -V -d myenv + spack env create -d rfm_spack_env + spack env activate -V -d rfm_spack_env + spack load bzip2@1.0.6 bzip2 --help From this point on, sanity and performance checking are exactly identical to any other ReFrame test. diff --git a/tutorials/build_systems/spack/spack_test.py b/tutorials/build_systems/spack/spack_test.py index 98a95d45ef..9a2cc082e8 100644 --- a/tutorials/build_systems/spack/spack_test.py +++ b/tutorials/build_systems/spack/spack_test.py @@ -18,7 +18,6 @@ class BZip2SpackCheck(rfm.RegressionTest): @run_before('compile') def setup_build_system(self): - self.build_system.environment = 'myenv' self.build_system.specs = ['bzip2@1.0.6'] @sanity_function diff --git a/tutorials/build_systems/spack/src/myenv/spack.yaml b/tutorials/build_systems/spack/src/myenv/spack.yaml deleted file mode 100644 index 61273fdde6..0000000000 --- a/tutorials/build_systems/spack/src/myenv/spack.yaml +++ /dev/null @@ -1,9 +0,0 @@ -spack: - specs: - - bzip2@1.0.6 - concretization: together - config: - install_tree: spack/opt/spack - module_roots: - tcl: spack/share/spack/modules - lmod: spack/share/spack/lmod From 88b4633cb99238d9f39cd2d63f7cca93e2cbe0a3 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Tue, 7 Sep 2021 19:58:25 +0200 Subject: [PATCH 5/5] Fine tune Spack tutorial text --- docs/tutorial_build_automation.rst | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/tutorial_build_automation.rst b/docs/tutorial_build_automation.rst index 6fa57c3077..51d3536739 100644 --- a/docs/tutorial_build_automation.rst +++ b/docs/tutorial_build_automation.rst @@ -109,14 +109,13 @@ Here is the test's code: When :attr:`~reframe.core.pipeline.RegressionTest.build_system` is set to ``'Spack'``, ReFrame will leverage Spack environments in order to build the test code. -If the environment is not specified by the user, ReFrame will automatically create one inside the stage directory. -That's the case in this example, where the software to be built is specified through the attribute :attr:`~reframe.core.buildsystems.Spack.specs` - -When Spack environments are specified by the user, ReFrame treats them as *test resources* so it expects to find them under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. +By default, ReFrame will create a new Spack environment in the test's stage directory and add the requested :attr:`~reframe.core.buildsystems.Spack.specs` to it. +Users may also specify an existing Spack environment by setting the :attr:`~reframe.core.buildsystems.Spack.environment` attribute. +In this case, ReFrame treats the environment as a *test resource* so it expects to find it under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. As with every other test, ReFrame will copy the test's resources to its stage directory before building it. -ReFrame will then activate the generated environment, add the given specs using the ``spack add`` command and install them. -Here is what ReFrame generates as a build script in this example: +ReFrame will then activate the generated environment (either the one provided by the user or the one generated by ReFrame), add the given specs using the ``spack add`` command and, finally, install the packages in the environment. +Here is what ReFrame generates as a build script for this example: .. code:: bash