diff --git a/src/deploy_tools/compare.py b/src/deploy_tools/compare.py index 61a0ce9..5123a06 100644 --- a/src/deploy_tools/compare.py +++ b/src/deploy_tools/compare.py @@ -87,10 +87,15 @@ def _reconstruct_deployment_config_from_modules(layout: Layout) -> Deployment: Note that the default versions will be different to those in initial configuration. """ releases = _collect_releases(layout) - default_versions = _collect_default_modulefile_versions(layout, list(releases)) - settings = DeploymentSettings(default_versions=default_versions) + deployment = Deployment(settings=DeploymentSettings(), releases=releases) - return Deployment(settings=settings, releases=releases) + # Only non-deprecated (live) modules are associated with a .version file + live_names = list(deployment.get_final_deployed_modules()) + deployment.settings.default_versions = _collect_default_modulefile_versions( + layout, live_names + ) + + return deployment def _collect_releases(layout: Layout) -> ReleasesByNameAndVersion: @@ -152,7 +157,12 @@ def _collect_default_modulefile_versions( default_versions: dict[str, str] = {} for name in names: - default_version = get_default_modulefile_version(name, layout) + try: + default_version = get_default_modulefile_version(name, layout) + except FileNotFoundError as exc: + raise ComparisonError( + f"Live module '{name}' has no default version file" + ) from exc if default_version is not None: default_versions[name] = default_version @@ -194,4 +204,6 @@ def _get_dict_diff(d1: dict[str, Any], d2: dict[str, Any]) -> str: def _yaml_dumps(obj: dict[str, Any], indent: int | None = None) -> str: ta = TypeAdapter(dict[str, Any]) - return yaml.safe_dump(ta.dump_python(obj), indent=indent) + # Dump in JSON mode so pydantic special types (e.g. AnyUrl) are coerced to YAML-safe + # primitives + return yaml.safe_dump(ta.dump_python(obj, mode="json"), indent=indent) diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index 939c13c..9ea3c58 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -1,5 +1,4 @@ import logging -import os import shutil from pathlib import Path @@ -69,7 +68,7 @@ def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None: mf_link = layout.get_modulefile_link(name, version, from_deprecated=deprecated) mf_link.parent.mkdir(parents=True, exist_ok=True) - os.symlink(modulefile, mf_link) + mf_link.symlink_to(modulefile) def _deploy_releases( @@ -155,6 +154,10 @@ def _delete_modulefiles_name_folder( ) -> None: modulefiles_name_path = layout.get_modulefiles_root(from_deprecated) / name + # Handle several versions of the same module being processed together + if not modulefiles_name_path.exists(): + return + # Ignore the default version file when checking for existing modulefile links if next(modulefiles_name_path.glob(MODULE_VERSIONS_GLOB), None) is None: shutil.rmtree(modulefiles_name_path) diff --git a/src/deploy_tools/layout.py b/src/deploy_tools/layout.py index 3d8b98d..e57a030 100644 --- a/src/deploy_tools/layout.py +++ b/src/deploy_tools/layout.py @@ -62,7 +62,6 @@ class Layout: DEFAULT_VERSION_FILENAME = ".version" DEPLOYMENT_SNAPSHOT_FILENAME = "deployment.yaml" - PREVIOUS_DEPLOYMENT_SNAPSHOT_FILENAME = "previous-deployment.yaml" def __init__(self, deployment_root: Path, build_root: Path | None = None) -> None: self._root = deployment_root diff --git a/src/deploy_tools/models/save_and_load.py b/src/deploy_tools/models/save_and_load.py index 818cd96..6aabf80 100644 --- a/src/deploy_tools/models/save_and_load.py +++ b/src/deploy_tools/models/save_and_load.py @@ -5,6 +5,7 @@ import yaml from pydantic import BaseModel +from pydantic import ValidationError as PydanticValidationError from ..errors import DeployToolsError from .deployment import ( @@ -67,8 +68,13 @@ def _load_release(path: Path) -> Release: if path.is_dir() or not path.suffix == YAML_FILE_SUFFIX: raise LoadError(f"Unexpected file in configuration directory:\n{path}") - with open(path) as f: - return load_from_yaml(Release, f) + try: + with open(path) as f: + return load_from_yaml(Release, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + # Include exc: under the top-level handler the traceback (and chained cause) is + # suppressed, so the failing field is only visible if surfaced in the message. + raise LoadError(f"Module configuration is invalid:\n{path}\n{exc}") from exc def _check_filepath_matches(version_path: Path, module: Module) -> None: @@ -76,9 +82,6 @@ def _check_filepath_matches(version_path: Path, module: Module) -> None: It should be the Module's name and version as ///.yaml """ - if version_path.is_dir() and version_path.suffix == YAML_FILE_SUFFIX: - raise LoadError(f"Module directory has incorrect suffix:\n{version_path}") - if not module.name == version_path.parent.name: raise LoadError( f"Module name {module.name} does not match path:\n{version_path}" diff --git a/src/deploy_tools/modulefile.py b/src/deploy_tools/modulefile.py index 5597de8..c884240 100644 --- a/src/deploy_tools/modulefile.py +++ b/src/deploy_tools/modulefile.py @@ -60,6 +60,8 @@ def apply_default_versions( overwrite=True, ) else: + # Defensive: remove a stale default file for a deployed module that has no + # entry in the default map. This ought to be unreachable via the CLI default_version_file.unlink(missing_ok=True) diff --git a/src/deploy_tools/snapshot.py b/src/deploy_tools/snapshot.py index ad2c254..4372851 100644 --- a/src/deploy_tools/snapshot.py +++ b/src/deploy_tools/snapshot.py @@ -1,7 +1,10 @@ import io import logging +from typing import cast -from git import Repo +import yaml +from git import BadName, Repo +from pydantic import ValidationError as PydanticValidationError from .errors import DeployToolsError from .layout import Layout @@ -54,14 +57,26 @@ def load_snapshot(layout: Layout, from_scratch: bool = False) -> Deployment: ) logger.debug("Loading snapshot: %s", layout.deployment_snapshot_path) - with open(layout.deployment_snapshot_path) as f: - return load_from_yaml(Deployment, f) + try: + with open(layout.deployment_snapshot_path) as f: + return load_from_yaml(Deployment, f) + except (OSError, yaml.YAMLError, PydanticValidationError, TypeError) as exc: + raise SnapshotError( + f"Deployment snapshot could not be read:\n{layout.deployment_snapshot_path}" + ) from exc def load_snapshot_from_ref(layout: Layout, ref: str) -> Deployment: """Load the deployment snapshot from the given git ref of the deployment area.""" logger.debug("Loading snapshot from ref: %s", ref) - repo = Repo(layout.deployment_root) - ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] - with io.BytesIO(ref_snapshot.data_stream.read()) as snapshot_f: # type: ignore - return load_from_yaml(Deployment, snapshot_f) + with Repo(layout.deployment_root) as repo: + try: + ref_snapshot = repo.commit(ref).tree[layout.DEPLOYMENT_SNAPSHOT_FILENAME] + except (BadName, KeyError) as exc: + raise SnapshotError( + f"Deployment snapshot not found at git ref:\n{ref}" + ) from exc + + snapshot_bytes = cast(bytes, ref_snapshot.data_stream.read()) + with io.BytesIO(snapshot_bytes) as snapshot_f: + return load_from_yaml(Deployment, snapshot_f) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index fb7d105..b2dd2dd 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -1,11 +1,12 @@ import logging from pathlib import Path -from git import Repo +from git import InvalidGitRepositoryError, Repo from .build import build, clean_build_area from .deploy import deploy_changes -from .layout import Layout +from .errors import DeployToolsError +from .layout import Layout, ModuleBuildLayout from .models.save_and_load import load_deployment from .snapshot import create_snapshot, load_snapshot from .templater import Templater, TemplateType @@ -15,7 +16,17 @@ logger = logging.getLogger(__name__) -IGNORE_DIRS = ["/build", "/modules/*/*/sif_files"] + +class SyncError(DeployToolsError): + """Raised when the deployment area is not in a state the sync process can use.""" + + +# Files not tracked in the deployment area's git repo (kept only as a change reference): +# the transient build area and the large Apptainer .sif images. +IGNORE_DIRS = [ + f"/{Layout.DEFAULT_BUILD_ROOT_NAME}", + f"/{Layout.MODULES_ROOT_NAME}/*/*/{ModuleBuildLayout.SIF_FILES_FOLDER}", +] def synchronise( @@ -45,18 +56,25 @@ def synchronise( logger.info("Creating git repository in deployment area") repo = _initialise_git_repo(layout.deployment_root, IGNORE_DIRS) else: - repo = Repo(layout.deployment_root) - - logger.info("Creating snapshot") - create_snapshot(deployment, layout) - - logger.info("Deploying changes") - deploy_changes(deployment_changes, layout) - - logger.info("Committing changes to git (for reference)") - repo.git.add("--all") - commit = repo.index.commit("Performed sync process") - logger.info("Commit SHA: %s", commit.hexsha) + try: + repo = Repo(layout.deployment_root) + except InvalidGitRepositoryError as exc: + raise SyncError( + f"Deployment area has a snapshot but is not a git repository " + f"(git metadata missing or corrupt):\n{layout.deployment_root}" + ) from exc + + with repo: + logger.info("Creating snapshot") + create_snapshot(deployment, layout) + + logger.info("Deploying changes") + deploy_changes(deployment_changes, layout) + + logger.info("Committing changes to git (for reference)") + repo.git.add("--all") + commit = repo.index.commit("Performed sync process") + logger.info("Commit SHA: %s", commit.hexsha) logger.info("Sync process finished") diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 84bdd53..7883a72 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -157,11 +157,11 @@ def _get_release_changes( return release_changes -def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None: +def _validate_added_modules(releases: list[Release], allow_all: bool) -> None: for release in releases: module = release.module if release.deprecated: - if not from_scratch: + if not allow_all: raise ValidationError( f"Module {module.name}/{module.version} cannot have deprecated " f"status on initial creation."