Replace dev mode with explicit configuration settings

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "jinja2",
     "pyyaml",
     "GitPython",
+    "natsort",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"

src/deploy_tools/deploy.py

@@ -51,9 +51,9 @@
     for release in to_remove:
         name = release.module.name
         version = release.module.version
+        deprecated = release.deprecated
 
-        from_deprecated = not release.module.is_dev_mode()
-        _remove_deployed_module(name, version, layout, from_deprecated)
+        _remove_deployed_module(name, version, layout, from_deprecated=deprecated)
 
 
 def _deploy_new_releases(to_add: list[Release], layout: Layout) -> None:
@@ -146,7 +146,7 @@
         _delete_modulefiles_name_folder(layout, release.module.name, True)
 
     for release in removed:
-        _delete_modulefiles_name_folder(layout, release.module.name, True)
+        _delete_modulefiles_name_folder(layout, release.module.name, release.deprecated)
         _delete_name_folder(release.module.name, layout.modules_root)
 
 

src/deploy_tools/models/deployment.py

@@ -1,10 +1,15 @@
-from .module import Release
+from collections import defaultdict
+
+from .module import Module, Release
 from .parent import ParentModel
 
 type ReleasesByVersion = dict[str, Release]
 type ReleasesByNameAndVersion = dict[str, ReleasesByVersion]
 type DefaultVersionsByName = dict[str, str]
 
+type ModulesByName = dict[str, list[Module]]
+type ModuleVersionsByName = dict[str, list[str]]
+
 
 class DeploymentSettings(ParentModel):
     """All global configuration settings for the Deployment."""
@@ -20,3 +25,33 @@ class Deployment(ParentModel):
 
     settings: DeploymentSettings
     releases: ReleasesByNameAndVersion
+
+    def get_final_deployed_modules(self) -> ModulesByName:
+        """Return modules that are expected to be deployed after a sync command.
+
+        This explicitly excludes any deprecated modules.
+        """
+        final_modules: ModulesByName = defaultdict(list)
+        for name, release_versions in self.releases.items():
+            modules = [
+                release.module
+                for release in release_versions.values()
+                if not release.deprecated
+            ]
+
+            if modules:
+                final_modules[name] = modules
+
+        return final_modules
+
+    def get_final_deployed_versions(self) -> ModuleVersionsByName:
+        """Return module versions that are expected to be deployed after a sync command.
+
+        This explicitly excludes any deprecated modules.
+        """
+        final_versions: ModuleVersionsByName = defaultdict(list)
+
+        for name, modules in self.get_final_deployed_modules().items():
+            final_versions[name] = [module.version for module in modules]
+
+        return final_versions

src/deploy_tools/models/module.py

@@ -13,8 +13,6 @@
     ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type")
 ]
 
-DEVELOPMENT_VERSION = "dev"
-
 
 class ModuleDependency(ParentModel):
     """Specify an Environment Module to include as a dependency.
@@ -47,9 +45,8 @@ class Module(ParentModel):
     dependencies: Sequence[ModuleDependency] = []
     env_vars: Sequence[EnvVar] = []
     applications: list[Application]
-
-    def is_dev_mode(self) -> bool:
-        return self.version == DEVELOPMENT_VERSION
+    allow_updates: bool = False
+    exclude_from_defaults: bool = False
 
 
 class Release(ParentModel):

src/deploy_tools/models/schemas/deployment.json

@@ -281,6 +281,16 @@
           },
           "title": "Applications",
           "type": "array"
+        },
+        "allow_updates": {
+          "default": false,
+          "title": "Allow Updates",
+          "type": "boolean"
+        },
+        "exclude_from_defaults": {
+          "default": false,
+          "title": "Exclude From Defaults",
+          "type": "boolean"
         }
       },
       "required": [

src/deploy_tools/models/schemas/module.json

@@ -263,6 +263,16 @@
           },
           "title": "Applications",
           "type": "array"
+        },
+        "allow_updates": {
+          "default": false,
+          "title": "Allow Updates",
+          "type": "boolean"
+        },
+        "exclude_from_defaults": {
+          "default": false,
+          "title": "Exclude From Defaults",
+          "type": "boolean"
         }
       },
       "required": [

src/deploy_tools/models/schemas/release.json

@@ -263,6 +263,16 @@
           },
           "title": "Applications",
           "type": "array"
+        },
+        "allow_updates": {
+          "default": false,
+          "title": "Allow Updates",
+          "type": "boolean"
+        },
+        "exclude_from_defaults": {
+          "default": false,
+          "title": "Exclude From Defaults",
+          "type": "boolean"
         }
       },
       "required": [

src/deploy_tools/modulefile.py

@@ -2,11 +2,9 @@
 from collections import defaultdict
 
 from .layout import Layout
-from .models.deployment import DefaultVersionsByName
+from .models.deployment import DefaultVersionsByName, ModuleVersionsByName
 from .templater import Templater, TemplateType
 
-type ModuleVersionsByName = dict[str, list[str]]
-
 VERSION_GLOB = f"*/[!{Layout.DEFAULT_VERSION_FILENAME}]*"
 
 DEFAULT_VERSION_REGEX = "^set ModulesVersion (.*)$"

src/deploy_tools/validate.py

@@ -1,22 +1,20 @@
 import logging
-from collections import defaultdict
-from copy import deepcopy
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
+from natsort import natsorted
+
 from .build import build
 from .layout import Layout
 from .models.changes import DeploymentChanges, ReleaseChanges
 from .models.deployment import (
     DefaultVersionsByName,
     Deployment,
+    ModulesByName,
     ReleasesByNameAndVersion,
 )
-from .models.module import DEVELOPMENT_VERSION, Release
+from .models.module import Release
 from .models.save_and_load import load_deployment
-from .modulefile import (
-    ModuleVersionsByName,
-)
 from .print_updates import print_updates
 from .snapshot import load_snapshot
 
@@ -91,42 +89,21 @@
     only valid for dependencies that are managed outside of the current deployment
     configuration.
     """
-    final_deployed_modules = _get_final_deployed_module_versions(deployment)
+    final_deployed_versions = deployment.get_final_deployed_versions()
 
     for name, release_versions in deployment.releases.items():
         for version, release in release_versions.items():
             for dependency in release.module.dependencies:
                 dep_name = dependency.name
                 dep_version = dependency.version
-                if dep_version is not None and dep_name in final_deployed_modules:
-                    if dep_version not in final_deployed_modules[dep_name]:
+                if dep_version is not None and dep_name in final_deployed_versions:
+                    if dep_version not in final_deployed_versions[dep_name]:
                         raise ValidationError(
                             f"Module {name}/{version} has unknown module dependency "
                             f"{dep_name}/{dep_version}."
                         )
 
 
-def _get_final_deployed_module_versions(
-    deployment: Deployment,
-) -> ModuleVersionsByName:
-    """Return module versions that will be deployed after sync action has completed.
-
-    This explicitly excludes any deprecated modules.
-    """
-    final_versions: ModuleVersionsByName = defaultdict(list)
-    for name, release_versions in deployment.releases.items():
-        versions = [
-            version
-            for version, release in release_versions.items()
-            if not release.deprecated
-        ]
-
-        if versions:
-            final_versions[name] = versions
-
-    return final_versions
-
-
 def _get_release_changes(
     old_releases: ReleasesByNameAndVersion,
     new_releases: ReleasesByNameAndVersion,
@@ -146,7 +123,7 @@
             old_release = old_releases[name][version]
 
             if old_release.module != new_release.module:
-                if new_release.module.is_dev_mode():
+                if old_release.module.allow_updates:
                     release_changes.to_update.append(new_release)
                     continue
 
@@ -169,8 +146,6 @@
                 release_changes.to_remove.append(old_release)
 
     _validate_added_modules(release_changes.to_add, allow_all)
-    _validate_updated_modules(release_changes.to_update)
-    _validate_deprecated_modules(release_changes.to_deprecate)
     _validate_removed_modules(release_changes.to_remove, allow_all)
 
     return release_changes
@@ -180,43 +155,17 @@
     for release in releases:
         module = release.module
         if release.deprecated:
-            if module.is_dev_mode():
-                raise ValidationError(
-                    f"Module {module.name}/{module.version} cannot be specified as"
-                    f"deprecated as it is in development mode."
-                )
-
             if not from_scratch:
                 raise ValidationError(
                     f"Module {module.name}/{module.version} cannot have deprecated "
                     f"status on initial creation."
                 )
 
 
-def _validate_updated_modules(releases: list[Release]) -> None:
-    for release in releases:
-        module = release.module
-        if release.deprecated:
-            raise ValidationError(
-                f"Module {module.name}/{module.version} cannot be specified as "
-                f"deprecated as it is in development mode."
-            )
-
-
-def _validate_deprecated_modules(releases: list[Release]) -> None:
-    for release in releases:
-        module = release.module
-        if module.is_dev_mode():
-            raise ValidationError(
-                f"Module {module.name}/{module.version} cannot be specified as "
-                f"deprecated as it is in development mode."
-            )
-
-
 def _validate_removed_modules(releases: list[Release], allow_all: bool) -> None:
     for release in releases:
         module = release.module
-        if not allow_all and not module.is_dev_mode() and not release.deprecated:
+        if not allow_all and not module.allow_updates and not release.deprecated:
             raise ValidationError(
                 f"Module {module.name}/{module.version} removed without prior "
                 f"deprecation."
@@ -225,25 +174,25 @@
 
 def validate_default_versions(deployment: Deployment) -> DefaultVersionsByName:
     """Validate configuration to get set of default version changes."""
-    final_deployed_modules = _get_final_deployed_module_versions(deployment)
+    final_deployed_versions = deployment.get_final_deployed_versions()
 
     for name, version in deployment.settings.default_versions.items():
-        if version not in final_deployed_modules[name]:
+        if version not in final_deployed_versions[name]:
             raise ValidationError(
                 f"Unable to configure {name}/{version} as default; module will not "
                 f"exist."
             )
 
     default_versions = _get_all_default_versions(
-        deployment.settings.default_versions, final_deployed_modules
+        deployment.settings.default_versions, deployment.get_final_deployed_modules()
     )
 
     return default_versions
 
 
 def _get_all_default_versions(
     initial_defaults: DefaultVersionsByName,
-    final_deployed_module_versions: ModuleVersionsByName,
+    final_deployed_modules: ModulesByName,
 ) -> DefaultVersionsByName:
     """Return the default versions that will be used for all modules in configuration.
 
@@ -254,15 +203,29 @@
     final_defaults: DefaultVersionsByName = {}
     final_defaults.update(initial_defaults)
 
-    for name in final_deployed_module_versions:
+    for name in final_deployed_modules:
         if name in final_defaults:
             continue
 
-        version_list = deepcopy(final_deployed_module_versions[name])
-        if DEVELOPMENT_VERSION in version_list:
-            version_list.remove(DEVELOPMENT_VERSION)
+        versions = [
+            module.version
+            for module in final_deployed_modules[name]
+            if not module.exclude_from_defaults
+        ]
+
+        if not versions:
+            # This prevents accidentally making a module that is not production-ready
+            # the default. Environment Modules will otherwise make an arbitrary module
+            # the default
+            raise ValidationError(
+                f"All modules require a default, but every version for name: {name} "
+                f"has set exclude_from_defaults=true. Please specify an explicit "
+                f"default or provide an alternative version without the exclusion."
+            )
 
-        version_list.sort()
-        final_defaults[name] = version_list[-1]
+        # The key follows natsort's documentation for supporting non-SemVer strings
+        # E.g. 1.2rc1 should come before 1.2.1 or 1.2
+        sorted_versions = natsorted(versions, key=lambda x: x.replace(".", "~") + "z")
+        final_defaults[name] = sorted_versions[-1]
 
     return final_defaults