From 2f432d67b94ab28e208757783e817bdb18cab102 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 17:06:41 +0000 Subject: [PATCH 1/6] Replace dev_mode with allow_updates configuration option for Releases --- src/deploy_tools/deploy.py | 6 +-- src/deploy_tools/models/module.py | 6 +-- .../models/schemas/deployment.json | 5 +++ src/deploy_tools/models/schemas/module.json | 5 +++ src/deploy_tools/models/schemas/release.json | 5 +++ src/deploy_tools/validate.py | 37 ++----------------- 6 files changed, 22 insertions(+), 42 deletions(-) diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index 5240902..d0c39f8 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -51,9 +51,9 @@ def _remove_releases(to_remove: list[Release], layout: Layout) -> None: for release in to_remove: name = release.module.name version = release.module.version + deprecated = release.deprecated - from_deprecated = not release.module.is_dev_mode() - _remove_deployed_module(name, version, layout, from_deprecated) + _remove_deployed_module(name, version, layout, from_deprecated=deprecated) def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None: @@ -146,7 +146,7 @@ def _remove_name_folders( _delete_modulefiles_name_folder(layout, release.module.name, True) for release in removed: - _delete_modulefiles_name_folder(layout, release.module.name, True) + _delete_modulefiles_name_folder(layout, release.module.name, release.deprecated) _delete_name_folder(release.module.name, layout.modules_root) diff --git a/src/deploy_tools/models/module.py b/src/deploy_tools/models/module.py index 6ff963b..046ff24 100644 --- a/src/deploy_tools/models/module.py +++ b/src/deploy_tools/models/module.py @@ -13,8 +13,6 @@ ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type") ] -DEVELOPMENT_VERSION = "dev" - class ModuleDependency(ParentModel): """Specify an Environment Module to include as a dependency. @@ -47,9 +45,7 @@ class Module(ParentModel): dependencies: Sequence[ModuleDependency] = [] env_vars: Sequence[EnvVar] = [] applications: list[Application] - - def is_dev_mode(self) -> bool: - return self.version == DEVELOPMENT_VERSION + allow_updates: bool = False class Release(ParentModel): diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index 96be029..c6dea1f 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -281,6 +281,11 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index e7488fe..d695beb 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -263,6 +263,11 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index e7488fe..d695beb 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -263,6 +263,11 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index a0f2bea..434aa6c 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -12,7 +12,7 @@ Deployment, ReleasesByNameAndVersion, ) -from .models.module import DEVELOPMENT_VERSION, Release +from .models.module import Release from .models.save_and_load import load_deployment from .modulefile import ( ModuleVersionsByName, @@ -146,7 +146,7 @@ def _get_release_changes( old_release = old_releases[name][version] if old_release.module != new_release.module: - if new_release.module.is_dev_mode(): + if old_release.module.allow_updates: release_changes.to_update.append(new_release) continue @@ -169,8 +169,6 @@ def _get_release_changes( release_changes.to_remove.append(old_release) _validate_added_modules(release_changes.to_add, allow_all) - _validate_updated_modules(release_changes.to_update) - _validate_deprecated_modules(release_changes.to_deprecate) _validate_removed_modules(release_changes.to_remove, allow_all) return release_changes @@ -180,12 +178,6 @@ def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None for release in releases: module = release.module if release.deprecated: - if module.is_dev_mode(): - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as" - f"deprecated as it is in development mode." - ) - if not from_scratch: raise ValidationError( f"Module {module.name}/{module.version} cannot have deprecated " @@ -193,30 +185,10 @@ def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None ) -def _validate_updated_modules(releases: list[Release]) -> None: - for release in releases: - module = release.module - if release.deprecated: - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as " - f"deprecated as it is in development mode." - ) - - -def _validate_deprecated_modules(releases: list[Release]) -> None: - for release in releases: - module = release.module - if module.is_dev_mode(): - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as " - f"deprecated as it is in development mode." - ) - - def _validate_removed_modules(releases: list[Release], allow_all: bool) -> None: for release in releases: module = release.module - if not allow_all and not module.is_dev_mode() and not release.deprecated: + if not allow_all and not module.allow_updates and not release.deprecated: raise ValidationError( f"Module {module.name}/{module.version} removed without prior " f"deprecation." @@ -259,9 +231,6 @@ def _get_all_default_versions( continue version_list = deepcopy(final_deployed_module_versions[name]) - if DEVELOPMENT_VERSION in version_list: - version_list.remove(DEVELOPMENT_VERSION) - version_list.sort() final_defaults[name] = version_list[-1] From 5e4a86eeb4d664303cb2f8f87a87cb9dd16e4433 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 17:10:45 +0000 Subject: [PATCH 2/6] Update samples directory to include allow_updates changes --- tests/samples/deploy-tools-output/deployment.yaml | 9 +++++++++ .../modules/argocd/v2.14.10/module.yaml | 1 + .../modules/dls-pmac-control/0.1/module.yaml | 1 + .../modules/dls-pmac-control/0.2/module.yaml | 1 + .../deploy-tools-output/modules/ec/i13-1/module.yaml | 1 + .../deploy-tools-output/modules/ec/p47/module.yaml | 1 + .../modules/edge-containers-cli/0.1/module.yaml | 1 + .../modules/example-module-apps/0.1/module.yaml | 1 + .../modules/example-module-deps/0.2/module.yaml | 1 + .../deploy-tools-output/modules/phoebus/0.1/module.yaml | 1 + 10 files changed, 18 insertions(+) diff --git a/tests/samples/deploy-tools-output/deployment.yaml b/tests/samples/deploy-tools-output/deployment.yaml index 4c7ce26..e033b54 100644 --- a/tests/samples/deploy-tools-output/deployment.yaml +++ b/tests/samples/deploy-tools-output/deployment.yaml @@ -3,6 +3,7 @@ releases: v2.14.10: deprecated: false module: + allow_updates: false applications: - app_type: binary hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa @@ -18,6 +19,7 @@ releases: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -50,6 +52,7 @@ releases: '0.2': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -83,6 +86,7 @@ releases: i13-1: deprecated: false module: + allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -100,6 +104,7 @@ releases: p47: deprecated: false module: + allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -118,6 +123,7 @@ releases: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -174,6 +180,7 @@ releases: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -221,6 +228,7 @@ releases: '0.2': deprecated: false module: + allow_updates: false applications: [] dependencies: - name: dls-pmac-control @@ -235,6 +243,7 @@ releases: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: diff --git a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml index ecc8a1e..e7a9b44 100644 --- a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml +++ b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: binary hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml index 6ba35a7..e9cb5e6 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml index 3b0c95b..7436505 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: diff --git a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml index 634e865..a45b70d 100644 --- a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: edge-containers-cli diff --git a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml index e1dcc07..a706143 100644 --- a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: edge-containers-cli diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml index 6ae6c5a..44b3078 100644 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml index 496c098..bf3d71e 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml index 7763904..5a161ad 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: dls-pmac-control diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml index c793ed1..8ba22f0 100644 --- a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: From 5c74e57720f03be107573a95de7c7c7d2667ca75 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 19:17:22 +0000 Subject: [PATCH 3/6] Use natsort to determine the default module to load --- pyproject.toml | 1 + src/deploy_tools/validate.py | 13 +++++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3f49d2b..d7757ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "jinja2", "pyyaml", "GitPython", + "natsort", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 434aa6c..3507fff 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -1,9 +1,10 @@ import logging from collections import defaultdict -from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory +from natsort import natsorted + from .build import build from .layout import Layout from .models.changes import DeploymentChanges, ReleaseChanges @@ -230,8 +231,12 @@ def _get_all_default_versions( if name in final_defaults: continue - version_list = deepcopy(final_deployed_module_versions[name]) - version_list.sort() - final_defaults[name] = version_list[-1] + # The key follows natsort's documentation for supporting non-SemVer strings + # E.g. 1.2rc1 should come before 1.2.1 or 1.2 + sorted_versions = natsorted( + final_deployed_module_versions[name], + key=lambda x: x.replace(".", "~") + "z", + ) + final_defaults[name] = sorted_versions[-1] return final_defaults From 788d9cc37667340a312ddf56727aeb43f3b51b64 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 20:56:40 +0000 Subject: [PATCH 4/6] Refactor get_deployed_module_versions() to be method in Deployment class --- src/deploy_tools/models/deployment.py | 37 ++++++++++++++++++++++++++- src/deploy_tools/modulefile.py | 4 +-- src/deploy_tools/validate.py | 34 +++++------------------- 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/src/deploy_tools/models/deployment.py b/src/deploy_tools/models/deployment.py index 8324c2d..c76c828 100644 --- a/src/deploy_tools/models/deployment.py +++ b/src/deploy_tools/models/deployment.py @@ -1,10 +1,15 @@ -from .module import Release +from collections import defaultdict + +from .module import Module, Release from .parent import ParentModel type ReleasesByVersion = dict[str, Release] type ReleasesByNameAndVersion = dict[str, ReleasesByVersion] type DefaultVersionsByName = dict[str, str] +type ModulesByName = dict[str, list[Module]] +type ModuleVersionsByName = dict[str, list[str]] + class DeploymentSettings(ParentModel): """All global configuration settings for the Deployment.""" @@ -20,3 +25,33 @@ class Deployment(ParentModel): settings: DeploymentSettings releases: ReleasesByNameAndVersion + + def get_final_deployed_modules(self) -> ModulesByName: + """Return modules that are expected to be deployed after a sync command. + + This explicitly excludes any deprecated modules. + """ + final_modules: ModulesByName = defaultdict(list) + for name, release_versions in self.releases.items(): + modules = [ + release.module + for release in release_versions.values() + if not release.deprecated + ] + + if modules: + final_modules[name] = modules + + return final_modules + + def get_final_deployed_versions(self) -> ModuleVersionsByName: + """Return module versions that are expected to be deployed after a sync command. + + This explicitly excludes any deprecated modules. + """ + final_versions: ModuleVersionsByName = defaultdict(list) + + for name, modules in self.get_final_deployed_modules().items(): + final_versions[name] = [module.version for module in modules] + + return final_versions diff --git a/src/deploy_tools/modulefile.py b/src/deploy_tools/modulefile.py index 3066d00..fe3df8c 100644 --- a/src/deploy_tools/modulefile.py +++ b/src/deploy_tools/modulefile.py @@ -2,11 +2,9 @@ from collections import defaultdict from .layout import Layout -from .models.deployment import DefaultVersionsByName +from .models.deployment import DefaultVersionsByName, ModuleVersionsByName from .templater import Templater, TemplateType -type ModuleVersionsByName = dict[str, list[str]] - VERSION_GLOB = f"*/[!{Layout.DEFAULT_VERSION_FILENAME}]*" DEFAULT_VERSION_REGEX = "^set ModulesVersion (.*)$" diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 3507fff..44d0e0a 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -1,5 +1,4 @@ import logging -from collections import defaultdict from pathlib import Path from tempfile import TemporaryDirectory @@ -92,42 +91,21 @@ def _validate_module_dependencies(deployment: Deployment) -> None: only valid for dependencies that are managed outside of the current deployment configuration. """ - final_deployed_modules = _get_final_deployed_module_versions(deployment) + final_deployed_versions = deployment.get_final_deployed_versions() for name, release_versions in deployment.releases.items(): for version, release in release_versions.items(): for dependency in release.module.dependencies: dep_name = dependency.name dep_version = dependency.version - if dep_version is not None and dep_name in final_deployed_modules: - if dep_version not in final_deployed_modules[dep_name]: + if dep_version is not None and dep_name in final_deployed_versions: + if dep_version not in final_deployed_versions[dep_name]: raise ValidationError( f"Module {name}/{version} has unknown module dependency " f"{dep_name}/{dep_version}." ) -def _get_final_deployed_module_versions( - deployment: Deployment, -) -> ModuleVersionsByName: - """Return module versions that will be deployed after sync action has completed. - - This explicitly excludes any deprecated modules. - """ - final_versions: ModuleVersionsByName = defaultdict(list) - for name, release_versions in deployment.releases.items(): - versions = [ - version - for version, release in release_versions.items() - if not release.deprecated - ] - - if versions: - final_versions[name] = versions - - return final_versions - - def _get_release_changes( old_releases: ReleasesByNameAndVersion, new_releases: ReleasesByNameAndVersion, @@ -198,17 +176,17 @@ def _validate_removed_modules(releases: list[Release], allow_all: bool) -> None: def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName: """Validate configuration to get set of default version changes.""" - final_deployed_modules = _get_final_deployed_module_versions(deployment) + final_deployed_versions = deployment.get_final_deployed_versions() for name, version in deployment.settings.default_versions.items(): - if version not in final_deployed_modules[name]: + if version not in final_deployed_versions[name]: raise ValidationError( f"Unable to configure {name}/{version} as default; module will not " f"exist." ) default_versions = _get_all_default_versions( - deployment.settings.default_versions, final_deployed_modules + deployment.settings.default_versions, final_deployed_versions ) return default_versions From e72e3b27a18786e8c8274df1e2d2de5699e0d886 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 21:28:17 +0000 Subject: [PATCH 5/6] Provide option to exclude a particular module version from becoming an implicit default --- src/deploy_tools/models/module.py | 1 + .../models/schemas/deployment.json | 5 +++ src/deploy_tools/models/schemas/module.json | 5 +++ src/deploy_tools/models/schemas/release.json | 5 +++ src/deploy_tools/validate.py | 31 +++++++++++++------ 5 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/deploy_tools/models/module.py b/src/deploy_tools/models/module.py index 046ff24..1d4ffc6 100644 --- a/src/deploy_tools/models/module.py +++ b/src/deploy_tools/models/module.py @@ -46,6 +46,7 @@ class Module(ParentModel): env_vars: Sequence[EnvVar] = [] applications: list[Application] allow_updates: bool = False + exclude_from_defaults: bool = False class Release(ParentModel): diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index c6dea1f..a0e4197 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -286,6 +286,11 @@ "default": false, "title": "Allow Updates", "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index d695beb..6c14e8a 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -268,6 +268,11 @@ "default": false, "title": "Allow Updates", "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index d695beb..6c14e8a 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -268,6 +268,11 @@ "default": false, "title": "Allow Updates", "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index 44d0e0a..96f5e7b 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -10,13 +10,11 @@ from .models.deployment import ( DefaultVersionsByName, Deployment, + ModulesByName, ReleasesByNameAndVersion, ) from .models.module import Release from .models.save_and_load import load_deployment -from .modulefile import ( - ModuleVersionsByName, -) from .print_updates import print_updates from .snapshot import load_snapshot @@ -186,7 +184,7 @@ def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName: ) default_versions = _get_all_default_versions( - deployment.settings.default_versions, final_deployed_versions + deployment.settings.default_versions, deployment.get_final_deployed_modules() ) return default_versions @@ -194,7 +192,7 @@ def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName: def _get_all_default_versions( initial_defaults: DefaultVersionsByName, - final_deployed_module_versions: ModuleVersionsByName, + final_deployed_modules: ModulesByName, ) -> DefaultVersionsByName: """Return the default versions that will be used for all modules in configuration. @@ -205,16 +203,29 @@ def _get_all_default_versions( final_defaults: DefaultVersionsByName = {} final_defaults.update(initial_defaults) - for name in final_deployed_module_versions: + for name in final_deployed_modules: if name in final_defaults: continue + versions = [ + module.version + for module in final_deployed_modules[name] + if not module.exclude_from_defaults + ] + + if not versions: + # This prevents accidentally making a module that is not production-ready + # the default. Environment Modules will otherwise make an arbitrary module + # the default + raise ValidationError( + f"All modules require a default, but every version for name: {name} " + f"has set exclude_from_defaults=true. Please specify an explicit " + f"default or provide an alternative version without the exclusion." + ) + # The key follows natsort's documentation for supporting non-SemVer strings # E.g. 1.2rc1 should come before 1.2.1 or 1.2 - sorted_versions = natsorted( - final_deployed_module_versions[name], - key=lambda x: x.replace(".", "~") + "z", - ) + sorted_versions = natsorted(versions, key=lambda x: x.replace(".", "~") + "z") final_defaults[name] = sorted_versions[-1] return final_defaults From 84c8d5145cd12a4e68b4e4db6835cf3e097c182c Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Thu, 3 Jul 2025 21:43:44 +0000 Subject: [PATCH 6/6] Update samples directory for automated system testing --- tests/samples/deploy-tools-output/deployment.yaml | 9 +++++++++ .../modules/argocd/v2.14.10/module.yaml | 1 + .../modules/dls-pmac-control/0.1/module.yaml | 1 + .../modules/dls-pmac-control/0.2/module.yaml | 1 + .../deploy-tools-output/modules/ec/i13-1/module.yaml | 1 + .../deploy-tools-output/modules/ec/p47/module.yaml | 1 + .../modules/edge-containers-cli/0.1/module.yaml | 1 + .../modules/example-module-apps/0.1/module.yaml | 1 + .../modules/example-module-deps/0.2/module.yaml | 1 + .../deploy-tools-output/modules/phoebus/0.1/module.yaml | 1 + 10 files changed, 18 insertions(+) diff --git a/tests/samples/deploy-tools-output/deployment.yaml b/tests/samples/deploy-tools-output/deployment.yaml index e033b54..773c954 100644 --- a/tests/samples/deploy-tools-output/deployment.yaml +++ b/tests/samples/deploy-tools-output/deployment.yaml @@ -13,6 +13,7 @@ releases: dependencies: [] description: Demonstration of binary download env_vars: [] + exclude_from_defaults: false name: argocd version: v2.14.10 dls-pmac-control: @@ -47,6 +48,7 @@ releases: env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.1 + exclude_from_defaults: false name: dls-pmac-control version: '0.1' '0.2': @@ -80,6 +82,7 @@ releases: env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.2 + exclude_from_defaults: false name: dls-pmac-control version: '0.2' ec: @@ -99,6 +102,7 @@ releases: value: i13-1-beamline/i13-1 - name: EC_SERVICES_REPO value: https://gitlab.diamond.ac.uk/controls/containers/beamline/i13-1-services.git + exclude_from_defaults: false name: ec version: i13-1 p47: @@ -117,6 +121,7 @@ releases: value: p47-beamline/p47 - name: EC_SERVICES_REPO value: https://github.com/epics-containers/p47-services + exclude_from_defaults: false name: ec version: p47 edge-containers-cli: @@ -174,6 +179,7 @@ releases: value: ARGOCD - name: EC_LOG_URL value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* + exclude_from_defaults: false name: edge-containers-cli version: '0.1' example-module-apps: @@ -222,6 +228,7 @@ releases: env_vars: - name: OTHER_VALUE value: Test message OTHER_VALUE from example-module-folder + exclude_from_defaults: false name: example-module-apps version: '0.1' example-module-deps: @@ -237,6 +244,7 @@ releases: version: '0.1' description: Demonstration of deploy-tools dependencies env_vars: [] + exclude_from_defaults: false name: example-module-deps version: '0.2' phoebus: @@ -265,6 +273,7 @@ releases: dependencies: [] description: Containerised release of CSS Phoebus env_vars: [] + exclude_from_defaults: false name: phoebus version: '0.1' settings: diff --git a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml index e7a9b44..398cac8 100644 --- a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml +++ b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml @@ -8,5 +8,6 @@ applications: dependencies: [] description: Demonstration of binary download env_vars: [] +exclude_from_defaults: false name: argocd version: v2.14.10 diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml index e9cb5e6..539e4a1 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml @@ -26,5 +26,6 @@ description: Demonstration of the deploy-tools process env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.1 +exclude_from_defaults: false name: dls-pmac-control version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml index 7436505..82c14d5 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml @@ -26,5 +26,6 @@ description: Demonstration of the deploy-tools process env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.2 +exclude_from_defaults: false name: dls-pmac-control version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml index a45b70d..a5b1857 100644 --- a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml @@ -11,5 +11,6 @@ env_vars: value: i13-1-beamline/i13-1 - name: EC_SERVICES_REPO value: https://gitlab.diamond.ac.uk/controls/containers/beamline/i13-1-services.git +exclude_from_defaults: false name: ec version: i13-1 diff --git a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml index a706143..e6acc3e 100644 --- a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml @@ -11,5 +11,6 @@ env_vars: value: p47-beamline/p47 - name: EC_SERVICES_REPO value: https://github.com/epics-containers/p47-services +exclude_from_defaults: false name: ec version: p47 diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml index 44b3078..e52be03 100644 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml @@ -49,5 +49,6 @@ env_vars: value: ARGOCD - name: EC_LOG_URL value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* +exclude_from_defaults: false name: edge-containers-cli version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml index bf3d71e..8cef07b 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml @@ -40,5 +40,6 @@ description: Demonstration of a module configuration folder env_vars: - name: OTHER_VALUE value: Test message OTHER_VALUE from example-module-folder +exclude_from_defaults: false name: example-module-apps version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml index 5a161ad..b0095fa 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml @@ -7,5 +7,6 @@ dependencies: version: '0.1' description: Demonstration of deploy-tools dependencies env_vars: [] +exclude_from_defaults: false name: example-module-deps version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml index 8ba22f0..c719778 100644 --- a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml @@ -20,5 +20,6 @@ applications: dependencies: [] description: Containerised release of CSS Phoebus env_vars: [] +exclude_from_defaults: false name: phoebus version: '0.1'