From 9441467720e2ff81cf2441bbf8205f201c385549 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 17:26:03 +0100 Subject: [PATCH 1/6] trivial test for duplicate keys --- unittests/test_schema.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 53dc2745..a591d055 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -200,3 +200,15 @@ def test_recipe_environments_yaml(recipe_paths): with open(p / "environments.yaml") as fid: raw = yaml.load(fid, Loader=yaml.Loader) schema.EnvironmentsValidator.validate(raw) + + +def test_unique_properties(): + invalid_config = dedent( + """ + name: invalid-config + name: duplicate-name + """ + ) + + with pytest.raises(Exception): + yaml.load(invalid_config, Loader=yaml.Loader) From 2bb1b1356a734a8a2514e617cd3b3d768c3b76b2 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 17:45:44 +0100 Subject: [PATCH 2/6] give a try to ruamel.yaml --- requirements.txt | 2 +- stackinator/builder.py | 6 ++++-- stackinator/cache.py | 6 ++++-- stackinator/recipe.py | 20 +++++++++++--------- test_stackinator.py | 2 +- unittests/test_schema.py | 40 +++++++++++++++++++--------------------- 6 files changed, 40 insertions(+), 36 deletions(-) diff --git a/requirements.txt b/requirements.txt index a5589617..01b82a57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Jinja2 jsonschema pytest -PyYAML +ruamel.yaml >=0.15.1, <0.18.0 diff --git a/stackinator/builder.py b/stackinator/builder.py index 53de2ba0..7cd03a3b 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -9,10 +9,12 @@ from datetime import datetime import jinja2 -import yaml +from ruamel.yaml import YAML from . import VERSION, cache, root_logger, spack_util +yaml = YAML() + def install(src, dst, *, ignore=None, symlinks=False): """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory. @@ -345,7 +347,7 @@ def generate(self, recipe): if repo_yaml.exists() and repo_yaml.is_file(): # open repos.yaml file and reat the list of repos with repo_yaml.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) P = raw["repos"] self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}") diff --git a/stackinator/cache.py b/stackinator/cache.py index 24177e33..773d0c2b 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -1,15 +1,17 @@ import os import pathlib -import yaml +from ruamel.yaml import YAML from . import schema +yaml = YAML() + def configuration_from_file(file, mount): with file.open() as fid: # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # validate the yaml schema.CacheValidator.validate(raw) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 82589955..7163bc5f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -3,11 +3,13 @@ import re import jinja2 -import yaml +from ruamel.yaml import YAML from . import cache, root_logger, schema, spack_util from .etc import envvars +yaml = YAML() + class Recipe: @property @@ -58,7 +60,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml") with compiler_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) self.generate_compiler_specs(raw) @@ -77,7 +79,7 @@ def __init__(self, args): self.packages = None if packages_path.is_file(): with packages_path.open() as fid: - self.packages = yaml.load(fid, Loader=yaml.Loader) + self.packages = yaml.load(fid) self._logger.debug("creating packages") @@ -86,7 +88,7 @@ def __init__(self, args): recipe_packages_path = self.path / "packages.yaml" if recipe_packages_path.is_file(): with recipe_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) recipe_packages = raw["packages"] # load system/packages.yaml -> system_packages (if it exists) @@ -95,7 +97,7 @@ def __init__(self, args): if system_packages_path.is_file(): # load system yaml with system_packages_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) system_packages = raw["packages"] # extract gcc packages from system packages @@ -115,7 +117,7 @@ def __init__(self, args): if network_path.is_file(): self._logger.debug(f"opening {network_path}") with network_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) if "packages" in raw: network_packages = raw["packages"] if "mpi" in raw: @@ -138,7 +140,7 @@ def __init__(self, args): raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml") with environments_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) # add a special environment that installs tools required later in the build process. # currently we only need squashfs for creating the squashfs file. raw["uenv_tools"] = { @@ -256,7 +258,7 @@ def config(self, config_path): raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml") with config_path.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) self._config = raw @@ -315,7 +317,7 @@ def environment_view_meta(self): @property def modules_yaml(self): with self.modules.open() as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix() return yaml.dump(raw) diff --git a/test_stackinator.py b/test_stackinator.py index a783d468..30a42dc5 100755 --- a/test_stackinator.py +++ b/test_stackinator.py @@ -5,7 +5,7 @@ # "jinja2", # "jsonschema", # "pytest", -# "pyYAML", +# "ruamel.yaml >=0.15.1, <0.18.0", # ] # /// diff --git a/unittests/test_schema.py b/unittests/test_schema.py index a591d055..baef7d1a 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -5,10 +5,12 @@ import jsonschema import pytest -import yaml +from ruamel.yaml import YAML import stackinator.schema as schema +yaml = YAML() + @pytest.fixture def test_path(): @@ -38,7 +40,7 @@ def recipe_paths(test_path, recipes): def test_config_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "config.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/user-environment" assert raw["spack"]["commit"] is None @@ -57,10 +59,7 @@ def test_config_yaml(yaml_path): repo: https://github.com/spack/spack.git commit: develop-packages """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] is None assert raw["spack"]["packages"]["commit"] is not None @@ -78,10 +77,7 @@ def test_config_yaml(yaml_path): packages: repo: https://github.com/spack/spack.git """) - raw = yaml.load( - config, - Loader=yaml.Loader, - ) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) assert raw["spack"]["commit"] == "develop" assert raw["spack"]["packages"]["commit"] is None @@ -91,7 +87,7 @@ def test_config_yaml(yaml_path): # full config with open(yaml_path / "config.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) assert raw["store"] == "/alternative-point" assert raw["spack"]["commit"] == "6408b51" @@ -107,7 +103,7 @@ def test_config_yaml(yaml_path): spack: repo: https://github.com/spack/spack.git """) - raw = yaml.load(config, Loader=yaml.Loader) + raw = yaml.load(config) schema.ConfigValidator.validate(raw) @@ -115,20 +111,20 @@ def test_recipe_config_yaml(recipe_paths): # validate the config.yaml in the test recipes for p in recipe_paths: with open(p / "config.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.ConfigValidator.validate(raw) def test_compilers_yaml(yaml_path): # test that the defaults are set as expected with open(yaml_path / "compilers.defaults.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "10.2"} assert raw["llvm"] is None with open(yaml_path / "compilers.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) assert raw["gcc"] == {"version": "11"} assert raw["llvm"] == {"version": "13"} @@ -139,13 +135,13 @@ def test_recipe_compilers_yaml(recipe_paths): # validate the compilers.yaml in the test recipes for p in recipe_paths: with open(p / "compilers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.CompilersValidator.validate(raw) def test_environments_yaml(yaml_path): with open(yaml_path / "environments.full.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) # the defaults-env does not set fields @@ -186,7 +182,7 @@ def test_environments_yaml(yaml_path): # check that only allowed fields are accepted # from an example that was silently validated with open(yaml_path / "environments.err-providers.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) with pytest.raises( jsonschema.exceptions.ValidationError, match=r"Additional properties are not allowed \('providers' was unexpected", @@ -198,7 +194,7 @@ def test_recipe_environments_yaml(recipe_paths): # validate the environments.yaml in the test recipes for p in recipe_paths: with open(p / "environments.yaml") as fid: - raw = yaml.load(fid, Loader=yaml.Loader) + raw = yaml.load(fid) schema.EnvironmentsValidator.validate(raw) @@ -210,5 +206,7 @@ def test_unique_properties(): """ ) - with pytest.raises(Exception): - yaml.load(invalid_config, Loader=yaml.Loader) + from ruamel.yaml.constructor import DuplicateKeyError + + with pytest.raises(DuplicateKeyError): + yaml.load(invalid_config) From 3cd58c4c6183a4caa771d36e26e913dbbfcff4e6 Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 17:57:22 +0100 Subject: [PATCH 3/6] sneak in a minor fix in doc and formatting --- stackinator/builder.py | 2 +- stackinator/recipe.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 7cd03a3b..3db2903c 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -141,7 +141,7 @@ def environment_meta(self, recipe): "root": /user-environment/env/default, "activate": /user-environment/env/default/activate.sh, "description": "simple devolpment env: compilers, MPI, python, cmake." - "env_vars": { + "recipe_variables": { ... } }, diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 7163bc5f..a004f2ac 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -292,17 +292,17 @@ def environment_view_meta(self): env.set_list(name, [], envvars.EnvVarOp.SET) else: env.set_scalar(name, value) + for v in ev_inputs["prepend_path"]: ((name, value),) = v.items() if not envvars.is_list_var(name): raise RuntimeError(f"{name} in the {view['name']} view is not a known prefix path variable") - env.set_list(name, [value], envvars.EnvVarOp.APPEND) + for v in ev_inputs["append_path"]: ((name, value),) = v.items() if not envvars.is_list_var(name): raise RuntimeError(f"{name} in the {view['name']} view is not a known prefix path variable") - env.set_list(name, [value], envvars.EnvVarOp.PREPEND) view_meta[view["name"]] = { @@ -408,6 +408,7 @@ def generate_environment_specs(self, raw): # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []} if view_config is None: view_config = {} + view_config.setdefault("link", "roots") view_config.setdefault("uenv", {}) view_config["uenv"].setdefault("add_compilers", True) From 4d0e1b922a47ea141b22c59fbdc9bb7ce6cbe54c Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Wed, 19 Nov 2025 18:07:11 +0100 Subject: [PATCH 4/6] fix dependency also for stack-config command --- bin/stack-config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/stack-config b/bin/stack-config index 66496991..9491f3a7 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -4,7 +4,7 @@ # dependencies = [ # "jinja2", # "jsonschema", -# "pyYAML", +# "ruamel.yaml >=0.15.1, <0.18.0", # ] # /// From 879f87877e3ab96cf366cf5d825469e4bd5d759f Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 20 Nov 2025 12:15:57 +0100 Subject: [PATCH 5/6] already using the new api --- bin/stack-config | 2 +- requirements.txt | 2 +- test_stackinator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/stack-config b/bin/stack-config index 9491f3a7..63c82bcc 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -4,7 +4,7 @@ # dependencies = [ # "jinja2", # "jsonschema", -# "ruamel.yaml >=0.15.1, <0.18.0", +# "ruamel.yaml >=0.15.1" # ] # /// diff --git a/requirements.txt b/requirements.txt index 01b82a57..79d70ed3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ Jinja2 jsonschema pytest -ruamel.yaml >=0.15.1, <0.18.0 +ruamel.yaml >=0.15.1 diff --git a/test_stackinator.py b/test_stackinator.py index 30a42dc5..8950b788 100755 --- a/test_stackinator.py +++ b/test_stackinator.py @@ -5,7 +5,7 @@ # "jinja2", # "jsonschema", # "pytest", -# "ruamel.yaml >=0.15.1, <0.18.0", +# "ruamel.yaml >=0.15.1", # ] # /// From f92a64cd1ee6c012307154a5d83eced942ab5cba Mon Sep 17 00:00:00 2001 From: Alberto Invernizzi Date: Thu, 20 Nov 2025 12:16:27 +0100 Subject: [PATCH 6/6] WIP: brutal solution for different dump API if we decide it is worth switching, i'm going to deal with this properly with the needed (small) refactoring --- stackinator/builder.py | 14 ++++++++------ stackinator/cache.py | 6 ++++-- stackinator/recipe.py | 6 +++--- stackinator/schema.py | 9 +++++++-- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 3db2903c..fd1c4ab6 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -308,17 +308,17 @@ def generate(self, recipe): # the packages.yaml configuration that will be used when building all environments # - the system packages.yaml with gcc removed # - plus additional packages provided by the recipe - global_packages_yaml = yaml.dump(recipe.packages["global"]) + global_packages_path = config_path / "packages.yaml" with global_packages_path.open("w") as fid: - fid.write(global_packages_yaml) + yaml.dump(recipe.packages["global"], fid) # generate a mirrors.yaml file if build caches have been configured if recipe.mirror: dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the build cache mirror: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + cache.generate_mirrors_yaml(recipe.mirror, fid) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to @@ -439,7 +439,10 @@ def generate(self, recipe): compiler_config_path.mkdir(exist_ok=True) for file, raw in files.items(): with (compiler_config_path / file).open(mode="w") as f: - f.write(raw) + if type(raw) is str: + f.write(raw) + else: + yaml.dump(raw, f) # generate the makefile and spack.yaml files that describe the environments environment_files = recipe.environment_files @@ -473,11 +476,10 @@ def generate(self, recipe): ) # write modules/modules.yaml - modules_yaml = recipe.modules_yaml generate_modules_path = self.path / "modules" generate_modules_path.mkdir(exist_ok=True) with (generate_modules_path / "modules.yaml").open("w") as f: - f.write(modules_yaml) + yaml.dump(recipe.modules_yaml_data, f) # write the meta data meta_path = store_path / "meta" diff --git a/stackinator/cache.py b/stackinator/cache.py index 773d0c2b..2adb80ac 100644 --- a/stackinator/cache.py +++ b/stackinator/cache.py @@ -42,7 +42,7 @@ def configuration_from_file(file, mount): return raw -def generate_mirrors_yaml(config): +def generate_mirrors_yaml(config, out): path = config["path"].as_posix() mirrors = { "mirrors": { @@ -57,4 +57,6 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) + yaml = YAML() + yaml.default_flow_style = True + yaml.dump(mirrors, out) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index a004f2ac..b9db773f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -315,11 +315,11 @@ def environment_view_meta(self): return view_meta @property - def modules_yaml(self): + def modules_yaml_data(self): with self.modules.open() as fid: raw = yaml.load(fid) raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix() - return yaml.dump(raw) + return raw # creates the self.environments field that describes the full specifications # for all of the environments sets, grouped in environments, from the raw @@ -514,7 +514,7 @@ def compiler_files(self): files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config) # compilers/gcc/packages.yaml if compiler == "gcc": - files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"]) + files["config"][compiler]["packages.yaml"] = self.packages["gcc"] return files diff --git a/stackinator/schema.py b/stackinator/schema.py index 4e981900..918d0803 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -3,7 +3,7 @@ from textwrap import dedent import jsonschema -import yaml +from ruamel.yaml import YAML from . import root_logger @@ -11,7 +11,12 @@ def py2yaml(data, indent): - dump = yaml.dump(data) + yaml = YAML() + from io import StringIO + + buffer = StringIO() + yaml.dump(data, buffer) + dump = buffer.getvalue() lines = [ln for ln in dump.split("\n") if ln != ""] res = ("\n" + " " * indent).join(lines) return res