From c59c03c1aa4e08a46bab07d1f0f643ebdb65e273 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Mon, 15 Jun 2026 18:13:01 +0000 Subject: [PATCH 01/13] Have compare command dump values in JSON mode to support special types --- src/deploy_tools/compare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/deploy_tools/compare.py b/src/deploy_tools/compare.py index 61a0ce9..16ddaa0 100644 --- a/src/deploy_tools/compare.py +++ b/src/deploy_tools/compare.py @@ -194,4 +194,6 @@ def _get_dict_diff(d1: dict[str, Any], d2: dict[str, Any]) -> str: def _yaml_dumps(obj: dict[str, Any], indent: int | None = None) -> str: ta = TypeAdapter(dict[str, Any]) - return yaml.safe_dump(ta.dump_python(obj), indent=indent) + # Dump in JSON mode so pydantic special types (e.g. AnyUrl) are coerced to YAML-safe + # primitives + return yaml.safe_dump(ta.dump_python(obj, mode="json"), indent=indent) From 58d4b740571c5d80575fc2e52d9c26fc03e726bb Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Mon, 15 Jun 2026 19:05:17 +0000 Subject: [PATCH 02/13] Only search non-deprecated modules for default versions in compare --- src/deploy_tools/compare.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/deploy_tools/compare.py b/src/deploy_tools/compare.py index 16ddaa0..dbdd943 100644 --- a/src/deploy_tools/compare.py +++ b/src/deploy_tools/compare.py @@ -87,10 +87,15 @@ def _reconstruct_deployment_config_from_modules(layout: Layout) -> Deployment: Note that the default versions will be different to those in initial configuration. """ releases = _collect_releases(layout) - default_versions = _collect_default_modulefile_versions(layout, list(releases)) - settings = DeploymentSettings(default_versions=default_versions) + deployment = Deployment(settings=DeploymentSettings(), releases=releases) - return Deployment(settings=settings, releases=releases) + # Only non-deprecated (live) modules are associated with a .version file + live_names = list(deployment.get_final_deployed_modules()) + deployment.settings.default_versions = _collect_default_modulefile_versions( + layout, live_names + ) + + return deployment def _collect_releases(layout: Layout) -> ReleasesByNameAndVersion: From 33d72d8355db9fc937923fd457591802f6409daf Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:30:09 +0000 Subject: [PATCH 03/13] Fix double-deprecation removal of name folders --- src/deploy_tools/deploy.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index 939c13c..e85220f 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -155,6 +155,10 @@ def _delete_modulefiles_name_folder( ) -> None: modulefiles_name_path = layout.get_modulefiles_root(from_deprecated) / name + # Handle several versions of the same module being processed together + if not modulefiles_name_path.exists(): + return + # Ignore the default version file when checking for existing modulefile links if next(modulefiles_name_path.glob(MODULE_VERSIONS_GLOB), None) is None: shutil.rmtree(modulefiles_name_path) From 6668421819356fbb2515c45dd34cae8570f12a80 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 17 Jun 2026 10:02:31 +0000 Subject: [PATCH 04/13] Remove previous snapshot dead code from Layout This feature was replaced with the git ref option to the compare command. --- src/deploy_tools/layout.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/deploy_tools/layout.py b/src/deploy_tools/layout.py index 3d8b98d..e57a030 100644 --- a/src/deploy_tools/layout.py +++ b/src/deploy_tools/layout.py @@ -62,7 +62,6 @@ class Layout: DEFAULT_VERSION_FILENAME = ".version" DEPLOYMENT_SNAPSHOT_FILENAME = "deployment.yaml" - PREVIOUS_DEPLOYMENT_SNAPSHOT_FILENAME = "previous-deployment.yaml" def __init__(self, deployment_root: Path, build_root: Path | None = None) -> None: self._root = deployment_root From 0bb2a8588397cc663f42ebfdd56868cdcd028e2c Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:31:08 +0000 Subject: [PATCH 05/13] Improve exception handling for compare command and snapshots --- src/deploy_tools/compare.py | 7 ++++++- src/deploy_tools/snapshot.py | 22 ++++++++++++++++++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/src/deploy_tools/compare.py b/src/deploy_tools/compare.py index dbdd943..5123a06 100644 --- a/src/deploy_tools/compare.py +++ b/src/deploy_tools/compare.py @@ -157,7 +157,12 @@ def _collect_default_modulefile_versions( default_versions: dict[str, str] = {} for name in names: - default_version = get_default_modulefile_version(name, layout) + try: + default_version = get_default_modulefile_version(name, layout) + except FileNotFoundError as exc: + raise ComparisonError( + f"Live module '{name}' has no default version file" + ) from exc if default_version is not None: default_versions[name] = default_version diff --git a/src/deploy_tools/snapshot.py b/src/deploy_tools/snapshot.py index ad2c254..e29fcc5 100644 --- a/src/deploy_tools/snapshot.py +++ b/src/deploy_tools/snapshot.py @@ -1,7 +1,9 @@ import io import logging -from git import Repo +import yaml +from git import BadName, Repo +from pydantic import ValidationError as PydanticValidationError from .errors import DeployToolsError from .layout import Layout @@ -54,14 +56,26 @@ def load_snapshot(layout: Layout, from_scratch: bool = False) -> Deployment: ) logger.debug("Loading snapshot: %s", layout.deployment_snapshot_path) - with open(layout.deployment_snapshot_path) as f: - return load_from_yaml(Deployment, f) + try: + with open(layout.deployment_snapshot_path) as f: + return load_from_yaml(Deployment, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + raise SnapshotError( + f"Deployment snapshot could not be read:\n{layout.deployment_snapshot_path}" + ) from exc def load_snapshot_from_ref(layout: Layout, ref: str) -> Deployment: """Load the deployment snapshot from the given git ref of the deployment area.""" logger.debug("Loading snapshot from ref: %s", ref) repo = Repo(layout.deployment_root) - ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] + + try: + ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] + except (BadName, KeyError) as exc: + raise SnapshotError( + f"Deployment snapshot not found at git ref:\n{ref}" + ) from exc + with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore return load_from_yaml(Deployment, snapshot_f) From 1a8dfc629240993514d4eef55f70bea2383c5d72 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:27:30 +0000 Subject: [PATCH 06/13] Fix typo in validate.py with from_scratch -> allow_all --- src/deploy_tools/validate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 84bdd53..7883a72 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -157,11 +157,11 @@ def _get_release_changes( return release_changes -def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None: +def _validate_added_modules(releases: list[Release], allow_all: bool) -> None: for release in releases: module = release.module if release.deprecated: - if not from_scratch: + if not allow_all: raise ValidationError( f"Module {module.name}/{module.version} cannot have deprecated " f"status on initial creation." From 283396ad23e1123703d4889a6948a9b57cb07498 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 17 Jun 2026 13:36:44 +0000 Subject: [PATCH 07/13] Use pathlib symlink_to instead of os function --- src/deploy_tools/deploy.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index e85220f..9ea3c58 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -1,5 +1,4 @@ import logging -import os import shutil from pathlib import Path @@ -69,7 +68,7 @@ def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None: mf_link = layout.get_modulefile_link(name, version, from_deprecated=deprecated) mf_link.parent.mkdir(parents=True, exist_ok=True) - os.symlink(modulefile, mf_link) + mf_link.symlink_to(modulefile) def _deploy_releases( From 20050c281464a136643807033113107c96c6e93f Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 17 Jun 2026 13:39:40 +0000 Subject: [PATCH 08/13] Use Layout variables to calculate .gitignore paths --- src/deploy_tools/sync.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index fb7d105..e17a6fc 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -5,7 +5,7 @@ from .build import build, clean_build_area from .deploy import deploy_changes -from .layout import Layout +from .layout import Layout, ModuleBuildLayout from .models.save_and_load import load_deployment from .snapshot import create_snapshot, load_snapshot from .templater import Templater, TemplateType @@ -15,7 +15,12 @@ logger = logging.getLogger(__name__) -IGNORE_DIRS = ["/build", "/modules/*/*/sif_files"] +# Files not tracked in the deployment area's git repo (kept only as a change reference): +# the transient build area and the large Apptainer .sif images. +IGNORE_DIRS = [ + f"/{Layout.DEFAULT_BUILD_ROOT_NAME}", + f"/{Layout.MODULES_ROOT_NAME}/*/*/{ModuleBuildLayout.SIF_FILES_FOLDER}", +] def synchronise( From b0d99016ec652c1252224ac451cbdd66088f0b31 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 17 Jun 2026 18:40:57 +0000 Subject: [PATCH 09/13] Ensure GitPython repos are closed properly --- src/deploy_tools/snapshot.py | 19 +++++++++---------- src/deploy_tools/sync.py | 17 +++++++++-------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/deploy_tools/snapshot.py b/src/deploy_tools/snapshot.py index e29fcc5..134bced 100644 --- a/src/deploy_tools/snapshot.py +++ b/src/deploy_tools/snapshot.py @@ -68,14 +68,13 @@ def load_snapshot(layout: Layout, from_scratch: bool = False) -> Deployment: def load_snapshot_from_ref(layout: Layout, ref: str) -> Deployment: """Load the deployment snapshot from the given git ref of the deployment area.""" logger.debug("Loading snapshot from ref: %s", ref) - repo = Repo(layout.deployment_root) - - try: - ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] - except (BadName, KeyError) as exc: - raise SnapshotError( - f"Deployment snapshot not found at git ref:\n{ref}" - ) from exc + with Repo(layout.deployment_root) as repo: + try: + ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] + except (BadName, KeyError) as exc: + raise SnapshotError( + f"Deployment snapshot not found at git ref:\n{ref}" + ) from exc - with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore - return load_from_yaml(Deployment, snapshot_f) + with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore + return load_from_yaml(Deployment, snapshot_f) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index e17a6fc..cb633fc 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -52,16 +52,17 @@ def synchronise( else: repo = Repo(layout.deployment_root) - logger.info("Creating snapshot") - create_snapshot(deployment, layout) + with repo: + logger.info("Creating snapshot") + create_snapshot(deployment, layout) - logger.info("Deploying changes") - deploy_changes(deployment_changes, layout) + logger.info("Deploying changes") + deploy_changes(deployment_changes, layout) - logger.info("Committing changes to git (for reference)") - repo.git.add("--all") - commit = repo.index.commit("Performed sync process") - logger.info("Commit SHA: %s", commit.hexsha) + logger.info("Committing changes to git (for reference)") + repo.git.add("--all") + commit = repo.index.commit("Performed sync process") + logger.info("Commit SHA: %s", commit.hexsha) logger.info("Sync process finished") From b6d4e40358eafe5032a125151b24847e3e19df9d Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:32:34 +0000 Subject: [PATCH 10/13] Add error handling when loading deployment configuration --- src/deploy_tools/sync.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index cb633fc..b2dd2dd 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -1,10 +1,11 @@ import logging from pathlib import Path -from git import Repo +from git import InvalidGitRepositoryError, Repo from .build import build, clean_build_area from .deploy import deploy_changes +from .errors import DeployToolsError from .layout import Layout, ModuleBuildLayout from .models.save_and_load import load_deployment from .snapshot import create_snapshot, load_snapshot @@ -15,6 +16,11 @@ logger = logging.getLogger(__name__) + +class SyncError(DeployToolsError): + """Raised when the deployment area is not in a state the sync process can use.""" + + # Files not tracked in the deployment area's git repo (kept only as a change reference): # the transient build area and the large Apptainer .sif images. IGNORE_DIRS = [ @@ -50,7 +56,13 @@ def synchronise( logger.info("Creating git repository in deployment area") repo = _initialise_git_repo(layout.deployment_root, IGNORE_DIRS) else: - repo = Repo(layout.deployment_root) + try: + repo = Repo(layout.deployment_root) + except InvalidGitRepositoryError as exc: + raise SyncError( + f"Deployment area has a snapshot but is not a git repository " + f"(git metadata missing or corrupt):\n{layout.deployment_root}" + ) from exc with repo: logger.info("Creating snapshot") From 55028fce31da295277e49f8531dbda8e8f7fa63b Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:33:39 +0000 Subject: [PATCH 11/13] Wrap load logic with standard deploy-tools error handling --- src/deploy_tools/models/save_and_load.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/deploy_tools/models/save_and_load.py b/src/deploy_tools/models/save_and_load.py index 818cd96..6aabf80 100644 --- a/src/deploy_tools/models/save_and_load.py +++ b/src/deploy_tools/models/save_and_load.py @@ -5,6 +5,7 @@ import yaml from pydantic import BaseModel +from pydantic import ValidationError as PydanticValidationError from ..errors import DeployToolsError from .deployment import ( @@ -67,8 +68,13 @@ def _load_release(path: Path) -> Release: if path.is_dir() or not path.suffix == YAML_FILE_SUFFIX: raise LoadError(f"Unexpected file in configuration directory:\n{path}") - with open(path) as f: - return load_from_yaml(Release, f) + try: + with open(path) as f: + return load_from_yaml(Release, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + # Include exc: under the top-level handler the traceback (and chained cause) is + # suppressed, so the failing field is only visible if surfaced in the message. + raise LoadError(f"Module configuration is invalid:\n{path}\n{exc}") from exc def _check_filepath_matches(version_path: Path, module: Module) -> None: @@ -76,9 +82,6 @@ def _check_filepath_matches(version_path: Path, module: Module) -> None: It should be the Module's name and version as ///.yaml """ - if version_path.is_dir() and version_path.suffix == YAML_FILE_SUFFIX: - raise LoadError(f"Module directory has incorrect suffix:\n{version_path}") - if not module.name == version_path.parent.name: raise LoadError( f"Module name {module.name} does not match path:\n{version_path}" From ad78bb87a8140fd2bd270655a46df36d3f458220 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 7 Jul 2026 15:34:31 +0000 Subject: [PATCH 12/13] Comment on defensive check in apply_default_versions() --- src/deploy_tools/modulefile.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/deploy_tools/modulefile.py b/src/deploy_tools/modulefile.py index 5597de8..c884240 100644 --- a/src/deploy_tools/modulefile.py +++ b/src/deploy_tools/modulefile.py @@ -60,6 +60,8 @@ def apply_default_versions( overwrite=True, ) else: + # Defensive: remove a stale default file for a deployed module that has no + # entry in the default map. This ought to be unreachable via the CLI default_version_file.unlink(missing_ok=True) From 67402951a7ce13bb963e03583aac641d905a59ca Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Wed, 8 Jul 2026 19:34:34 +0000 Subject: [PATCH 13/13] Cast returned gitdb data to bytes for type-hinting --- src/deploy_tools/snapshot.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/deploy_tools/snapshot.py b/src/deploy_tools/snapshot.py index 134bced..4372851 100644 --- a/src/deploy_tools/snapshot.py +++ b/src/deploy_tools/snapshot.py @@ -1,5 +1,6 @@ import io import logging +from typing import cast import yaml from git import BadName, Repo @@ -76,5 +77,6 @@ def load_snapshot_from_ref(layout: Layout, ref: str) -> Deployment: f"Deployment snapshot not found at git ref:\n{ref}" ) from exc - with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore + snapshot_bytes = cast(bytes, ref_snapshot.data_stream.read()) + with io.BytesIO(snapshot_bytes) as snapshot_f: return load_from_yaml(Deployment, snapshot_f)