diff --git a/pyproject.toml b/pyproject.toml index 3f49d2b..d7757ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "jinja2", "pyyaml", "GitPython", + "natsort", ] # Add project dependencies here, e.g. ["click", "numpy"] dynamic = ["version"] license.file = "LICENSE" diff --git a/src/deploy_tools/deploy.py b/src/deploy_tools/deploy.py index 5240902..d0c39f8 100644 --- a/src/deploy_tools/deploy.py +++ b/src/deploy_tools/deploy.py @@ -51,9 +51,9 @@ def _remove_releases(to_remove: list[Release], layout: Layout) -> None: for release in to_remove: name = release.module.name version = release.module.version + deprecated = release.deprecated - from_deprecated = not release.module.is_dev_mode() - _remove_deployed_module(name, version, layout, from_deprecated) + _remove_deployed_module(name, version, layout, from_deprecated=deprecated) def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None: @@ -146,7 +146,7 @@ def _remove_name_folders( _delete_modulefiles_name_folder(layout, release.module.name, True) for release in removed: - _delete_modulefiles_name_folder(layout, release.module.name, True) + _delete_modulefiles_name_folder(layout, release.module.name, release.deprecated) _delete_name_folder(release.module.name, layout.modules_root) diff --git a/src/deploy_tools/models/deployment.py b/src/deploy_tools/models/deployment.py index 8324c2d..c76c828 100644 --- a/src/deploy_tools/models/deployment.py +++ b/src/deploy_tools/models/deployment.py @@ -1,10 +1,15 @@ -from .module import Release +from collections import defaultdict + +from .module import Module, Release from .parent import ParentModel type ReleasesByVersion = dict[str, Release] type ReleasesByNameAndVersion = dict[str, ReleasesByVersion] type DefaultVersionsByName = dict[str, str] +type ModulesByName = dict[str, list[Module]] +type ModuleVersionsByName = dict[str, list[str]] + class DeploymentSettings(ParentModel): """All global configuration settings for the Deployment.""" @@ -20,3 +25,33 @@ class Deployment(ParentModel): settings: DeploymentSettings releases: ReleasesByNameAndVersion + + def get_final_deployed_modules(self) -> ModulesByName: + """Return modules that are expected to be deployed after a sync command. + + This explicitly excludes any deprecated modules. + """ + final_modules: ModulesByName = defaultdict(list) + for name, release_versions in self.releases.items(): + modules = [ + release.module + for release in release_versions.values() + if not release.deprecated + ] + + if modules: + final_modules[name] = modules + + return final_modules + + def get_final_deployed_versions(self) -> ModuleVersionsByName: + """Return module versions that are expected to be deployed after a sync command. + + This explicitly excludes any deprecated modules. + """ + final_versions: ModuleVersionsByName = defaultdict(list) + + for name, modules in self.get_final_deployed_modules().items(): + final_versions[name] = [module.version for module in modules] + + return final_versions diff --git a/src/deploy_tools/models/module.py b/src/deploy_tools/models/module.py index 6ff963b..1d4ffc6 100644 --- a/src/deploy_tools/models/module.py +++ b/src/deploy_tools/models/module.py @@ -13,8 +13,6 @@ ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type") ] -DEVELOPMENT_VERSION = "dev" - class ModuleDependency(ParentModel): """Specify an Environment Module to include as a dependency. @@ -47,9 +45,8 @@ class Module(ParentModel): dependencies: Sequence[ModuleDependency] = [] env_vars: Sequence[EnvVar] = [] applications: list[Application] - - def is_dev_mode(self) -> bool: - return self.version == DEVELOPMENT_VERSION + allow_updates: bool = False + exclude_from_defaults: bool = False class Release(ParentModel): diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index 96be029..a0e4197 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -281,6 +281,16 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index e7488fe..6c14e8a 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -263,6 +263,16 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index e7488fe..6c14e8a 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -263,6 +263,16 @@ }, "title": "Applications", "type": "array" + }, + "allow_updates": { + "default": false, + "title": "Allow Updates", + "type": "boolean" + }, + "exclude_from_defaults": { + "default": false, + "title": "Exclude From Defaults", + "type": "boolean" } }, "required": [ diff --git a/src/deploy_tools/modulefile.py b/src/deploy_tools/modulefile.py index 3066d00..fe3df8c 100644 --- a/src/deploy_tools/modulefile.py +++ b/src/deploy_tools/modulefile.py @@ -2,11 +2,9 @@ from collections import defaultdict from .layout import Layout -from .models.deployment import DefaultVersionsByName +from .models.deployment import DefaultVersionsByName, ModuleVersionsByName from .templater import Templater, TemplateType -type ModuleVersionsByName = dict[str, list[str]] - VERSION_GLOB = f"*/[!{Layout.DEFAULT_VERSION_FILENAME}]*" DEFAULT_VERSION_REGEX = "^set ModulesVersion (.*)$" diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index a0f2bea..96f5e7b 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -1,22 +1,20 @@ import logging -from collections import defaultdict -from copy import deepcopy from pathlib import Path from tempfile import TemporaryDirectory +from natsort import natsorted + from .build import build from .layout import Layout from .models.changes import DeploymentChanges, ReleaseChanges from .models.deployment import ( DefaultVersionsByName, Deployment, + ModulesByName, ReleasesByNameAndVersion, ) -from .models.module import DEVELOPMENT_VERSION, Release +from .models.module import Release from .models.save_and_load import load_deployment -from .modulefile import ( - ModuleVersionsByName, -) from .print_updates import print_updates from .snapshot import load_snapshot @@ -91,42 +89,21 @@ def _validate_module_dependencies(deployment: Deployment) -> None: only valid for dependencies that are managed outside of the current deployment configuration. """ - final_deployed_modules = _get_final_deployed_module_versions(deployment) + final_deployed_versions = deployment.get_final_deployed_versions() for name, release_versions in deployment.releases.items(): for version, release in release_versions.items(): for dependency in release.module.dependencies: dep_name = dependency.name dep_version = dependency.version - if dep_version is not None and dep_name in final_deployed_modules: - if dep_version not in final_deployed_modules[dep_name]: + if dep_version is not None and dep_name in final_deployed_versions: + if dep_version not in final_deployed_versions[dep_name]: raise ValidationError( f"Module {name}/{version} has unknown module dependency " f"{dep_name}/{dep_version}." ) -def _get_final_deployed_module_versions( - deployment: Deployment, -) -> ModuleVersionsByName: - """Return module versions that will be deployed after sync action has completed. - - This explicitly excludes any deprecated modules. - """ - final_versions: ModuleVersionsByName = defaultdict(list) - for name, release_versions in deployment.releases.items(): - versions = [ - version - for version, release in release_versions.items() - if not release.deprecated - ] - - if versions: - final_versions[name] = versions - - return final_versions - - def _get_release_changes( old_releases: ReleasesByNameAndVersion, new_releases: ReleasesByNameAndVersion, @@ -146,7 +123,7 @@ def _get_release_changes( old_release = old_releases[name][version] if old_release.module != new_release.module: - if new_release.module.is_dev_mode(): + if old_release.module.allow_updates: release_changes.to_update.append(new_release) continue @@ -169,8 +146,6 @@ def _get_release_changes( release_changes.to_remove.append(old_release) _validate_added_modules(release_changes.to_add, allow_all) - _validate_updated_modules(release_changes.to_update) - _validate_deprecated_modules(release_changes.to_deprecate) _validate_removed_modules(release_changes.to_remove, allow_all) return release_changes @@ -180,12 +155,6 @@ def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None for release in releases: module = release.module if release.deprecated: - if module.is_dev_mode(): - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as" - f"deprecated as it is in development mode." - ) - if not from_scratch: raise ValidationError( f"Module {module.name}/{module.version} cannot have deprecated " @@ -193,30 +162,10 @@ def _validate_added_modules(releases: list[Release], from_scratch: bool) -> None ) -def _validate_updated_modules(releases: list[Release]) -> None: - for release in releases: - module = release.module - if release.deprecated: - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as " - f"deprecated as it is in development mode." - ) - - -def _validate_deprecated_modules(releases: list[Release]) -> None: - for release in releases: - module = release.module - if module.is_dev_mode(): - raise ValidationError( - f"Module {module.name}/{module.version} cannot be specified as " - f"deprecated as it is in development mode." - ) - - def _validate_removed_modules(releases: list[Release], allow_all: bool) -> None: for release in releases: module = release.module - if not allow_all and not module.is_dev_mode() and not release.deprecated: + if not allow_all and not module.allow_updates and not release.deprecated: raise ValidationError( f"Module {module.name}/{module.version} removed without prior " f"deprecation." @@ -225,17 +174,17 @@ def _validate_removed_modules(releases: list[Release], allow_all: bool) -> None: def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName: """Validate configuration to get set of default version changes.""" - final_deployed_modules = _get_final_deployed_module_versions(deployment) + final_deployed_versions = deployment.get_final_deployed_versions() for name, version in deployment.settings.default_versions.items(): - if version not in final_deployed_modules[name]: + if version not in final_deployed_versions[name]: raise ValidationError( f"Unable to configure {name}/{version} as default; module will not " f"exist." ) default_versions = _get_all_default_versions( - deployment.settings.default_versions, final_deployed_modules + deployment.settings.default_versions, deployment.get_final_deployed_modules() ) return default_versions @@ -243,7 +192,7 @@ def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName: def _get_all_default_versions( initial_defaults: DefaultVersionsByName, - final_deployed_module_versions: ModuleVersionsByName, + final_deployed_modules: ModulesByName, ) -> DefaultVersionsByName: """Return the default versions that will be used for all modules in configuration. @@ -254,15 +203,29 @@ def _get_all_default_versions( final_defaults: DefaultVersionsByName = {} final_defaults.update(initial_defaults) - for name in final_deployed_module_versions: + for name in final_deployed_modules: if name in final_defaults: continue - version_list = deepcopy(final_deployed_module_versions[name]) - if DEVELOPMENT_VERSION in version_list: - version_list.remove(DEVELOPMENT_VERSION) + versions = [ + module.version + for module in final_deployed_modules[name] + if not module.exclude_from_defaults + ] + + if not versions: + # This prevents accidentally making a module that is not production-ready + # the default. Environment Modules will otherwise make an arbitrary module + # the default + raise ValidationError( + f"All modules require a default, but every version for name: {name} " + f"has set exclude_from_defaults=true. Please specify an explicit " + f"default or provide an alternative version without the exclusion." + ) - version_list.sort() - final_defaults[name] = version_list[-1] + # The key follows natsort's documentation for supporting non-SemVer strings + # E.g. 1.2rc1 should come before 1.2.1 or 1.2 + sorted_versions = natsorted(versions, key=lambda x: x.replace(".", "~") + "z") + final_defaults[name] = sorted_versions[-1] return final_defaults diff --git a/tests/samples/deploy-tools-output/deployment.yaml b/tests/samples/deploy-tools-output/deployment.yaml index 4c7ce26..773c954 100644 --- a/tests/samples/deploy-tools-output/deployment.yaml +++ b/tests/samples/deploy-tools-output/deployment.yaml @@ -3,6 +3,7 @@ releases: v2.14.10: deprecated: false module: + allow_updates: false applications: - app_type: binary hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa @@ -12,12 +13,14 @@ releases: dependencies: [] description: Demonstration of binary download env_vars: [] + exclude_from_defaults: false name: argocd version: v2.14.10 dls-pmac-control: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -45,11 +48,13 @@ releases: env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.1 + exclude_from_defaults: false name: dls-pmac-control version: '0.1' '0.2': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -77,12 +82,14 @@ releases: env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.2 + exclude_from_defaults: false name: dls-pmac-control version: '0.2' ec: i13-1: deprecated: false module: + allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -95,11 +102,13 @@ releases: value: i13-1-beamline/i13-1 - name: EC_SERVICES_REPO value: https://gitlab.diamond.ac.uk/controls/containers/beamline/i13-1-services.git + exclude_from_defaults: false name: ec version: i13-1 p47: deprecated: false module: + allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -112,12 +121,14 @@ releases: value: p47-beamline/p47 - name: EC_SERVICES_REPO value: https://github.com/epics-containers/p47-services + exclude_from_defaults: false name: ec version: p47 edge-containers-cli: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -168,12 +179,14 @@ releases: value: ARGOCD - name: EC_LOG_URL value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* + exclude_from_defaults: false name: edge-containers-cli version: '0.1' example-module-apps: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -215,12 +228,14 @@ releases: env_vars: - name: OTHER_VALUE value: Test message OTHER_VALUE from example-module-folder + exclude_from_defaults: false name: example-module-apps version: '0.1' example-module-deps: '0.2': deprecated: false module: + allow_updates: false applications: [] dependencies: - name: dls-pmac-control @@ -229,12 +244,14 @@ releases: version: '0.1' description: Demonstration of deploy-tools dependencies env_vars: [] + exclude_from_defaults: false name: example-module-deps version: '0.2' phoebus: '0.1': deprecated: false module: + allow_updates: false applications: - app_type: apptainer container: @@ -256,6 +273,7 @@ releases: dependencies: [] description: Containerised release of CSS Phoebus env_vars: [] + exclude_from_defaults: false name: phoebus version: '0.1' settings: diff --git a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml index ecc8a1e..398cac8 100644 --- a/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml +++ b/tests/samples/deploy-tools-output/modules/argocd/v2.14.10/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: binary hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa @@ -7,5 +8,6 @@ applications: dependencies: [] description: Demonstration of binary download env_vars: [] +exclude_from_defaults: false name: argocd version: v2.14.10 diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml index 6ba35a7..539e4a1 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: @@ -25,5 +26,6 @@ description: Demonstration of the deploy-tools process env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.1 +exclude_from_defaults: false name: dls-pmac-control version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml index 3b0c95b..82c14d5 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: @@ -25,5 +26,6 @@ description: Demonstration of the deploy-tools process env_vars: - name: EXAMPLE_VALUE value: Test message EXAMPLE_VALUE from example-module-file version 0.2 +exclude_from_defaults: false name: dls-pmac-control version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml index 634e865..a5b1857 100644 --- a/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/i13-1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -10,5 +11,6 @@ env_vars: value: i13-1-beamline/i13-1 - name: EC_SERVICES_REPO value: https://gitlab.diamond.ac.uk/controls/containers/beamline/i13-1-services.git +exclude_from_defaults: false name: ec version: i13-1 diff --git a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml index e1dcc07..e6acc3e 100644 --- a/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml +++ b/tests/samples/deploy-tools-output/modules/ec/p47/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: edge-containers-cli @@ -10,5 +11,6 @@ env_vars: value: p47-beamline/p47 - name: EC_SERVICES_REPO value: https://github.com/epics-containers/p47-services +exclude_from_defaults: false name: ec version: p47 diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml index 6ae6c5a..e52be03 100644 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: @@ -48,5 +49,6 @@ env_vars: value: ARGOCD - name: EC_LOG_URL value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* +exclude_from_defaults: false name: edge-containers-cli version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml index 496c098..8cef07b 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: @@ -39,5 +40,6 @@ description: Demonstration of a module configuration folder env_vars: - name: OTHER_VALUE value: Test message OTHER_VALUE from example-module-folder +exclude_from_defaults: false name: example-module-apps version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml index 7763904..b0095fa 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: [] dependencies: - name: dls-pmac-control @@ -6,5 +7,6 @@ dependencies: version: '0.1' description: Demonstration of deploy-tools dependencies env_vars: [] +exclude_from_defaults: false name: example-module-deps version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml index c793ed1..c719778 100644 --- a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml @@ -1,3 +1,4 @@ +allow_updates: false applications: - app_type: apptainer container: @@ -19,5 +20,6 @@ applications: dependencies: [] description: Containerised release of CSS Phoebus env_vars: [] +exclude_from_defaults: false name: phoebus version: '0.1'