Merge pull request #451 from pepkit/yacman-update

Update to yacman 0.9.3 to prepare for the new YAMLConfigManager

looper/conductor.py

@@ -21,7 +21,7 @@
 from pipestat import PipestatError
 from ubiquerg import expandpath, is_command_callable
 from yaml import dump
-from yacman import YAMLConfigManager
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
 
 from .const import *
 from .exceptions import JobSubmissionException, SampleFailedException

looper/divvy.py

@@ -9,8 +9,8 @@
 
 
 from shutil import copytree
-from yacman import YAMLConfigManager as YAMLConfigManager
-from yacman import FILEPATH_KEY, load_yaml, select_config
+from yacman import FutureYAMLConfigManager as YAMLConfigManager
+from yacman import write_lock, FILEPATH_KEY, load_yaml, select_config
 from yaml import SafeLoader
 from ubiquerg import is_writable, VersionInHelpParser
 
@@ -47,35 +47,30 @@ class ComputingConfiguration(YAMLConfigManager):
         `DIVCFG` file)
     """
 
-    def __init__(self, entries=None, filepath=None):
-        if not entries and not filepath:
-            # Handle the case of an empty one, when we'll use the default
-            filepath = select_divvy_config(None)
-
-        super(ComputingConfiguration, self).__init__(
-            entries=entries,
-            filepath=filepath,
-            schema_source=DEFAULT_CONFIG_SCHEMA,
-            validate_on_write=True,
+    def __init__(
+        self,
+        entries=None,
+        wait_max=None,
+        strict_ro_locks=False,
+        schema_source=None,
+        validate_on_write=False,
+    ):
+        super().__init__(
+            entries, wait_max, strict_ro_locks, schema_source, validate_on_write
         )
 
-        if not "compute_packages" in self:
-            raise Exception(
-                "Your divvy config file is not in divvy config format "
-                "(it lacks a compute_packages section): '{}'".format(filepath)
-            )
-            # We require that compute_packages be present, even if empty
+        if "compute_packages" not in self:
             self["compute_packages"] = {}
-
         # Initialize default compute settings.
         _LOGGER.debug("Establishing project compute settings")
         self.compute = None
         self.setdefault("adapters", None)
         self.activate_package(DEFAULT_COMPUTE_RESOURCES_NAME)
-        self.config_file = self.filepath
 
     def write(self, filename=None):
-        super(ComputingConfiguration, self).write(filepath=filename, exclude_case=True)
+        with write_lock(self) as locked_ym:
+            locked_ym.rebase()
+            locked_ym.write()
         filename = filename or getattr(self, FILEPATH_KEY)
         filedir = os.path.dirname(filename)
         # For this object, we *also* have to write the template files
@@ -159,7 +154,7 @@ def activate_package(self, package_name):
                     "Adding entries for package_name '{}'".format(package_name)
                 )
 
-            self.compute.update(self["compute_packages"][package_name])
+            self.compute.update_from_obj(self["compute_packages"][package_name])
 
             # Ensure submission template is absolute. This *used to be* handled
             # at update (so the paths were stored as absolutes in the packages),
@@ -168,7 +163,7 @@ def activate_package(self, package_name):
             if not os.path.isabs(self.compute["submission_template"]):
                 try:
                     self.compute["submission_template"] = os.path.join(
-                        os.path.dirname(self.filepath),
+                        os.path.dirname(self.default_config_file),
                         self.compute["submission_template"],
                     )
                 except AttributeError as e:

looper/project.py

@@ -144,7 +144,7 @@ def __init__(self, cfg=None, amendments=None, divcfg_path=None, **kwargs):
         self.dcc = (
             None
             if divcfg_path is None
-            else ComputingConfiguration(filepath=divcfg_path)
+            else ComputingConfiguration.from_yaml_file(filepath=divcfg_path)
         )
         if DRY_RUN_KEY in self and not self[DRY_RUN_KEY]:
             _LOGGER.debug("Ensuring project directories exist")

requirements/requirements-all.txt

@@ -6,8 +6,8 @@ logmuse>=0.2.0
 pandas>=2.0.2
 pephubclient>=0.1.2
 peppy>=0.40.0
-pipestat>=0.8.0
+pipestat>=0.8.2a1
 pyyaml>=3.12
 rich>=9.10.0
 ubiquerg>=0.5.2
-yacman>=0.9.2
+yacman==0.9.3

tests/divvytests/conftest.py

@@ -3,11 +3,17 @@
 import looper.divvy as divvy
 import pytest
 
+from looper.divvy import select_divvy_config, DEFAULT_CONFIG_SCHEMA
+
 
 THIS_DIR = os.path.dirname(os.path.abspath(__file__))
 DATA_DIR = os.path.join(THIS_DIR, "data/divcfg-master")
 FILES = glob.glob(DATA_DIR + "/*.yaml")
-DCC_ATTRIBUTES = divvy.ComputingConfiguration().keys()
+
+
+dcc_filepath = select_divvy_config(None)
+DCC = divvy.ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
+DCC_ATTRIBUTES = DCC.keys()
 
 
 @pytest.fixture
@@ -19,7 +25,7 @@ def empty_dcc():
 @pytest.fixture(params=FILES)
 def dcc(request):
     """Provide ComputingConfiguration objects for all files in divcfg repository"""
-    return divvy.ComputingConfiguration(filepath=request.param)
+    return divvy.ComputingConfiguration.from_yaml_file(filepath=request.param)
 
 
 @pytest.fixture

tests/divvytests/divvy_tests/test_divvy.py

@@ -6,7 +6,7 @@
 from tests.divvytests.conftest import DCC_ATTRIBUTES, FILES, mock_env_missing
 
 
-class DefaultDCCTests:
+class TestDefaultDCC:
     """Tests the default divvy.ComputingConfiguration object creation"""
 
     def test_no_args(self, empty_dcc):
@@ -22,7 +22,7 @@ def test_no_env_var(self, mock_env_missing, empty_dcc):
         empty_dcc
 
 
-class DCCTests:
+class TestDCC:
     """Tests the divvy.ComputingConfiguration object creation"""
 
     def test_object_creation(self, dcc):
@@ -35,7 +35,7 @@ def test_attrs_produced(self, att, dcc):
         dcc[att]
 
 
-class ActivatingTests:
+class TestActivating:
     """Test for the activate_package method"""
 
     def test_activating_default_package(self, dcc):
@@ -56,7 +56,7 @@ def test_not_activating_faulty_package(self, dcc, package):
         assert not dcc.activate_package(package)
 
 
-class GettingActivePackageTests:
+class TestGettingActivePackage:
     """Test for the get_active_package method"""
 
     def test_settings_nonempty(self, dcc):
@@ -65,7 +65,7 @@ def test_settings_nonempty(self, dcc):
         assert settings != YacAttMap()
 
 
-class ListingPackagesTests:
+class TestListingPackages:
     """Test for the list_compute_packages method"""
 
     def test_list_compute_packages_is_set(self, dcc):
@@ -77,7 +77,7 @@ def test_list_compute_packages_result_nonempty(self, dcc):
         assert dcc.list_compute_packages() != set()
 
 
-class ResettingSettingsTests:
+class TestResettingSettings:
     """ " Test for the reset_active_settings method"""
 
     def test_reset_active_settings(self, dcc):

tests/divvytests/regression/test_write_script.py

@@ -3,7 +3,7 @@
 from copy import deepcopy
 import random
 import pytest
-from looper.divvy import ComputingConfiguration
+from looper.divvy import ComputingConfiguration, select_divvy_config
 from tests.divvytests.helpers import get_random_key
 
 __author__ = "Vince Reuter"
@@ -19,7 +19,8 @@
 )
 def test_write_script_is_effect_free(tmpdir, extras):
     """Writing script doesn't change computing configuration."""
-    cc = ComputingConfiguration()
+    dcc_filepath = select_divvy_config(None)
+    cc = ComputingConfiguration.from_yaml_file(filepath=dcc_filepath)
     compute1 = deepcopy(cc["compute_packages"])
     cc.write_script(tmpdir.join(get_random_key(20) + ".sh").strpath, extras)
     assert cc["compute_packages"] == compute1

tests/divvytests/test_divvy_simple.py

@@ -4,6 +4,7 @@
 from collections import OrderedDict
 
 from yacman import YacAttMap
+from divvy import select_divvy_config
 
 # For interactive debugging:
 # import logmuse
@@ -12,7 +13,8 @@
 
 class TestPackageAtivation:
     def test_activate_package(self):
-        dcc = divvy.ComputingConfiguration()
+        dcc_filepath = select_divvy_config(None)
+        dcc = divvy.ComputingConfiguration().from_yaml_file(filepath=dcc_filepath)
         dcc.activate_package("default")
         t = dcc.compute["submission_template"]
         t2 = dcc["compute_packages"]["default"]["submission_template"]
@@ -25,7 +27,8 @@ def test_activate_package(self):
 
 class TestWriting:
     def test_write_script(self):
-        dcc = divvy.ComputingConfiguration()
+        dcc_filepath = select_divvy_config(None)
+        dcc = divvy.ComputingConfiguration().from_yaml_file(filepath=dcc_filepath)
         dcc
         dcc.activate_package("singularity_slurm")
         extra_vars = {