From 8222d09dfdf23852340f5b8954287549c0738402 Mon Sep 17 00:00:00 2001 From: Jack Morrison Date: Mon, 6 Mar 2023 17:23:36 -0500 Subject: [PATCH 1/6] Support arbitrary Spack environment configuration --- reframe/core/buildsystems.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 2ee6a8c64b..f33d6e3a1b 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -869,6 +869,12 @@ class Spack(BuildSystem): #: :default: ``[]`` install_opts = variable(typ.List[str], value=[]) + #: A list of Spack configurations in flattened YAML. + #: + #: :type: :class:`List[str]` + #: :default: ``[]`` + config_opts = variable(typ.List[str], value=[]) + def __init__(self): # Set to True if the environment was auto-generated self._auto_env = False @@ -881,6 +887,10 @@ def emit_build_commands(self, environ): ret.append(f'spack -e {self.environment} config add ' f'"config:install_tree:root:{install_tree}"') + if self.config_opts: + for opt in self.config_opts: + ret.append(f'spack -e {self.environment} config add "{opt}"') + if self.specs: specs_str = ' '.join(self.specs) ret.append(f'spack -e {self.environment} add {specs_str}') From 9721b131a0473f1326f27850bfc690572e090189 Mon Sep 17 00:00:00 2001 From: Jack Morrison Date: Wed, 8 Mar 2023 14:54:40 -0500 Subject: [PATCH 2/6] Include install_tree as part of Spack build system config_opts --- reframe/core/buildsystems.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index f33d6e3a1b..29256c982c 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -882,14 +882,14 @@ def __init__(self): def emit_build_commands(self, environ): ret = self._create_env_cmds() + config_opts = [] if self._auto_env: install_tree = self.install_tree or 'opt/spack' - ret.append(f'spack -e {self.environment} config add ' - f'"config:install_tree:root:{install_tree}"') + config_opts.append(f'config:install_tree:root:{install_tree}') - if self.config_opts: - for opt in self.config_opts: - ret.append(f'spack -e {self.environment} config add "{opt}"') + config_opts += self.config_opts + for opt in config_opts: + ret.append(f'spack -e {self.environment} config add "{opt}"') if self.specs: specs_str = ' '.join(self.specs) From 48d5706ade21e842fad708a53e68bbeeeb37fa95 Mon Sep 17 00:00:00 2001 From: Jack Morrison Date: Wed, 8 Mar 2023 14:59:03 -0500 Subject: [PATCH 3/6] Add unit test for Spack build system use of config_opts --- unittests/test_buildsystems.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/unittests/test_buildsystems.py b/unittests/test_buildsystems.py index 8350acbff9..2ee6d0ba1e 100644 --- a/unittests/test_buildsystems.py +++ b/unittests/test_buildsystems.py @@ -302,6 +302,19 @@ def test_spack_no_env(environ, tmp_path): assert build_system.environment == 'rfm_spack_env' +def test_spack_env_config(environ, tmp_path): + build_system = bs.Spack() + build_system.config_opts = ['section1:header1:value1', 'section2:header2:value2'] + with osext.change_dir(tmp_path): + assert build_system.emit_build_commands(environ) == [ + 'spack env create -d rfm_spack_env', + 'spack -e rfm_spack_env config add "config:install_tree:root:opt/spack"', + 'spack -e rfm_spack_env config add "section1:header1:value1"', + 'spack -e rfm_spack_env config add "section2:header2:value2"', + 'spack -e rfm_spack_env install', + ] + + def test_easybuild(environ, tmp_path): build_system = bs.EasyBuild() build_system.easyconfigs = ['ec1.eb', 'ec2.eb'] From f9bc8e62db33804fcf8b6923fd04052a4edf578d Mon Sep 17 00:00:00 2001 From: Jack Morrison Date: Wed, 8 Mar 2023 15:50:18 -0500 Subject: [PATCH 4/6] Add note about attribute to Spack build system tutorial --- docs/tutorial_build_automation.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/tutorial_build_automation.rst b/docs/tutorial_build_automation.rst index a656cfc136..a2d9db5ca9 100644 --- a/docs/tutorial_build_automation.rst +++ b/docs/tutorial_build_automation.rst @@ -108,6 +108,8 @@ By default, ReFrame will create a new Spack environment in the test's stage dire .. note:: Optional spec attributes, such as ``target`` and ``os``, should be specified in :attr:`~reframe.core.buildsystems.Spack.specs` and not as install options in :attr:`~reframe.core.buildsystems.Spack.install_opts`. +You can set Spack configuration options for the new environment with the :attr:`~reframe.core.buildsystems.Spack.config_opts` attribute. These options take precedence over Spack's ``spack.yaml`` defaults. + Users may also specify an existing Spack environment by setting the :attr:`~reframe.core.buildsystems.Spack.environment` attribute. In this case, ReFrame treats the environment as a *test resource* so it expects to find it under the test's :attr:`~reframe.core.pipeline.RegressionTest.sourcesdir`, which defaults to ``'src'``. From d56d515662acfa9106764d17f76cdeda82ec6b08 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 8 Mar 2023 23:07:15 +0100 Subject: [PATCH 5/6] Minor style changes --- unittests/test_buildsystems.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unittests/test_buildsystems.py b/unittests/test_buildsystems.py index 2ee6d0ba1e..e34092f30f 100644 --- a/unittests/test_buildsystems.py +++ b/unittests/test_buildsystems.py @@ -304,11 +304,12 @@ def test_spack_no_env(environ, tmp_path): def test_spack_env_config(environ, tmp_path): build_system = bs.Spack() - build_system.config_opts = ['section1:header1:value1', 'section2:header2:value2'] + build_system.config_opts = ['section1:header1:value1', + 'section2:header2:value2'] with osext.change_dir(tmp_path): assert build_system.emit_build_commands(environ) == [ 'spack env create -d rfm_spack_env', - 'spack -e rfm_spack_env config add "config:install_tree:root:opt/spack"', + 'spack -e rfm_spack_env config add "config:install_tree:root:opt/spack"', # noqa: E501 'spack -e rfm_spack_env config add "section1:header1:value1"', 'spack -e rfm_spack_env config add "section2:header2:value2"', 'spack -e rfm_spack_env install', From 9b8b548d50ec03a66f5f99e374d49ee9fba4e785 Mon Sep 17 00:00:00 2001 From: Vasileios Karakasis Date: Wed, 8 Mar 2023 23:22:31 +0100 Subject: [PATCH 6/6] Add version annotation in docs --- reframe/core/buildsystems.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/reframe/core/buildsystems.py b/reframe/core/buildsystems.py index 29256c982c..b1ec437896 100644 --- a/reframe/core/buildsystems.py +++ b/reframe/core/buildsystems.py @@ -873,6 +873,8 @@ class Spack(BuildSystem): #: #: :type: :class:`List[str]` #: :default: ``[]`` + #: + #: .. versionadded:: 4.2 config_opts = variable(typ.List[str], value=[]) def __init__(self): @@ -889,7 +891,7 @@ def emit_build_commands(self, environ): config_opts += self.config_opts for opt in config_opts: - ret.append(f'spack -e {self.environment} config add "{opt}"') + ret.append(f'spack -e {self.environment} config add "{opt}"') if self.specs: specs_str = ' '.join(self.specs)