yaml: change dependency from pyyaml to ruamel.yaml

bin/stack-config

@@ -4,7 +4,7 @@
 # dependencies = [
 #   "jinja2",
 #   "jsonschema",
-#   "pyYAML",
+#   "ruamel.yaml >=0.15.1"
 # ]
 # ///
 

requirements.txt

@@ -1,4 +1,4 @@
 Jinja2
 jsonschema
 pytest
-PyYAML
+ruamel.yaml >=0.15.1

stackinator/builder.py

@@ -9,10 +9,12 @@
 from datetime import datetime
 
 import jinja2
-import yaml
+from ruamel.yaml import YAML
 
 from . import VERSION, cache, root_logger, spack_util
 
+yaml = YAML()
+
 
 def install(src, dst, *, ignore=None, symlinks=False):
     """Call shutil.copytree or shutil.copy2. copy2 is used if `src` is not a directory.
@@ -139,7 +141,7 @@ def environment_meta(self, recipe):
               "root": /user-environment/env/default,
               "activate": /user-environment/env/default/activate.sh,
               "description": "simple devolpment env: compilers, MPI, python, cmake."
-              "env_vars": {
+              "recipe_variables": {
                 ...
               }
             },
@@ -306,17 +308,17 @@ def generate(self, recipe):
         # the packages.yaml configuration that will be used when building all environments
         # - the system packages.yaml with gcc removed
         # - plus additional packages provided by the recipe
-        global_packages_yaml = yaml.dump(recipe.packages["global"])
+
         global_packages_path = config_path / "packages.yaml"
         with global_packages_path.open("w") as fid:
-            fid.write(global_packages_yaml)
+            yaml.dump(recipe.packages["global"], fid)
 
         # generate a mirrors.yaml file if build caches have been configured
         if recipe.mirror:
             dst = config_path / "mirrors.yaml"
             self._logger.debug(f"generate the build cache mirror: {dst}")
             with dst.open("w") as fid:
-                fid.write(cache.generate_mirrors_yaml(recipe.mirror))
+                cache.generate_mirrors_yaml(recipe.mirror, fid)
 
         # Add custom spack package recipes, configured via Spack repos.
         # Step 1: copy Spack repos to store_path where they will be used to
@@ -345,7 +347,7 @@ def generate(self, recipe):
         if repo_yaml.exists() and repo_yaml.is_file():
             # open repos.yaml file and reat the list of repos
             with repo_yaml.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 P = raw["repos"]
 
             self._logger.debug(f"the system configuration has a repo file {repo_yaml} refers to {P}")
@@ -437,7 +439,10 @@ def generate(self, recipe):
             compiler_config_path.mkdir(exist_ok=True)
             for file, raw in files.items():
                 with (compiler_config_path / file).open(mode="w") as f:
-                    f.write(raw)
+                    if type(raw) is str:
+                        f.write(raw)
+                    else:
+                        yaml.dump(raw, f)
 
         # generate the makefile and spack.yaml files that describe the environments
         environment_files = recipe.environment_files
@@ -471,11 +476,10 @@ def generate(self, recipe):
             )
 
         # write modules/modules.yaml
-        modules_yaml = recipe.modules_yaml
         generate_modules_path = self.path / "modules"
         generate_modules_path.mkdir(exist_ok=True)
         with (generate_modules_path / "modules.yaml").open("w") as f:
-            f.write(modules_yaml)
+            yaml.dump(recipe.modules_yaml_data, f)
 
         # write the meta data
         meta_path = store_path / "meta"

stackinator/cache.py

@@ -1,15 +1,17 @@
 import os
 import pathlib
 
-import yaml
+from ruamel.yaml import YAML
 
 from . import schema
 
+yaml = YAML()
+
 
 def configuration_from_file(file, mount):
     with file.open() as fid:
         # load the raw yaml input
-        raw = yaml.load(fid, Loader=yaml.Loader)
+        raw = yaml.load(fid)
 
         # validate the yaml
         schema.CacheValidator.validate(raw)
@@ -40,7 +42,7 @@ def configuration_from_file(file, mount):
         return raw
 
 
-def generate_mirrors_yaml(config):
+def generate_mirrors_yaml(config, out):
     path = config["path"].as_posix()
     mirrors = {
         "mirrors": {
@@ -55,4 +57,6 @@ def generate_mirrors_yaml(config):
         }
     }
 
-    return yaml.dump(mirrors, default_flow_style=False)
+    yaml = YAML()
+    yaml.default_flow_style = True
+    yaml.dump(mirrors, out)

stackinator/recipe.py

@@ -3,11 +3,13 @@
 import re
 
 import jinja2
-import yaml
+from ruamel.yaml import YAML
 
 from . import cache, root_logger, schema, spack_util
 from .etc import envvars
 
+yaml = YAML()
+
 
 class Recipe:
     @property
@@ -58,7 +60,7 @@ def __init__(self, args):
             raise FileNotFoundError(f"The recipe path '{compiler_path}' does not contain compilers.yaml")
 
         with compiler_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.CompilersValidator.validate(raw)
             self.generate_compiler_specs(raw)
 
@@ -77,7 +79,7 @@ def __init__(self, args):
         self.packages = None
         if packages_path.is_file():
             with packages_path.open() as fid:
-                self.packages = yaml.load(fid, Loader=yaml.Loader)
+                self.packages = yaml.load(fid)
 
         self._logger.debug("creating packages")
 
@@ -86,7 +88,7 @@ def __init__(self, args):
         recipe_packages_path = self.path / "packages.yaml"
         if recipe_packages_path.is_file():
             with recipe_packages_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 recipe_packages = raw["packages"]
 
         # load system/packages.yaml -> system_packages (if it exists)
@@ -95,7 +97,7 @@ def __init__(self, args):
         if system_packages_path.is_file():
             # load system yaml
             with system_packages_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 system_packages = raw["packages"]
 
         # extract gcc packages from system packages
@@ -115,7 +117,7 @@ def __init__(self, args):
         if network_path.is_file():
             self._logger.debug(f"opening {network_path}")
             with network_path.open() as fid:
-                raw = yaml.load(fid, Loader=yaml.Loader)
+                raw = yaml.load(fid)
                 if "packages" in raw:
                     network_packages = raw["packages"]
                 if "mpi" in raw:
@@ -138,7 +140,7 @@ def __init__(self, args):
             raise FileNotFoundError(f"The recipe path '{environments_path}' does not contain environments.yaml")
 
         with environments_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             # add a special environment that installs tools required later in the build process.
             # currently we only need squashfs for creating the squashfs file.
             raw["uenv_tools"] = {
@@ -256,7 +258,7 @@ def config(self, config_path):
             raise FileNotFoundError(f"The recipe path '{config_path}' does not contain config.yaml")
 
         with config_path.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             schema.ConfigValidator.validate(raw)
             self._config = raw
 
@@ -290,17 +292,17 @@ def environment_view_meta(self):
                             env.set_list(name, [], envvars.EnvVarOp.SET)
                         else:
                             env.set_scalar(name, value)
+
                 for v in ev_inputs["prepend_path"]:
                     ((name, value),) = v.items()
                     if not envvars.is_list_var(name):
                         raise RuntimeError(f"{name} in the {view['name']} view is not a known prefix path variable")
-
                     env.set_list(name, [value], envvars.EnvVarOp.APPEND)
+
                 for v in ev_inputs["append_path"]:
                     ((name, value),) = v.items()
                     if not envvars.is_list_var(name):
                         raise RuntimeError(f"{name} in the {view['name']} view is not a known prefix path variable")
-
                     env.set_list(name, [value], envvars.EnvVarOp.PREPEND)
 
                 view_meta[view["name"]] = {
@@ -313,11 +315,11 @@ def environment_view_meta(self):
         return view_meta
 
     @property
-    def modules_yaml(self):
+    def modules_yaml_data(self):
         with self.modules.open() as fid:
-            raw = yaml.load(fid, Loader=yaml.Loader)
+            raw = yaml.load(fid)
             raw["modules"]["default"]["roots"]["tcl"] = (pathlib.Path(self.mount) / "modules").as_posix()
-            return yaml.dump(raw)
+            return raw
 
     # creates the self.environments field that describes the full specifications
     # for all of the environments sets, grouped in environments, from the raw
@@ -406,6 +408,7 @@ def generate_environment_specs(self, raw):
                 # ["uenv"]["env_vars"] = {"set": [], "unset": [], "prepend_path": [], "append_path": []}
                 if view_config is None:
                     view_config = {}
+
                 view_config.setdefault("link", "roots")
                 view_config.setdefault("uenv", {})
                 view_config["uenv"].setdefault("add_compilers", True)
@@ -511,7 +514,7 @@ def compiler_files(self):
             files["config"][compiler]["spack.yaml"] = spack_yaml_template.render(config=config)
             # compilers/gcc/packages.yaml
             if compiler == "gcc":
-                files["config"][compiler]["packages.yaml"] = yaml.dump(self.packages["gcc"])
+                files["config"][compiler]["packages.yaml"] = self.packages["gcc"]
 
         return files
 

stackinator/schema.py

@@ -3,15 +3,20 @@
 from textwrap import dedent
 
 import jsonschema
-import yaml
+from ruamel.yaml import YAML
 
 from . import root_logger
 
 prefix = pathlib.Path(__file__).parent.resolve()
 
 
 def py2yaml(data, indent):
-    dump = yaml.dump(data)
+    yaml = YAML()
+    from io import StringIO
+
+    buffer = StringIO()
+    yaml.dump(data, buffer)
+    dump = buffer.getvalue()
     lines = [ln for ln in dump.split("\n") if ln != ""]
     res = ("\n" + " " * indent).join(lines)
     return res

test_stackinator.py

@@ -5,7 +5,7 @@
 #   "jinja2",
 #   "jsonschema",
 #   "pytest",
-#   "pyYAML",
+#   "ruamel.yaml >=0.15.1",
 # ]
 # ///