From 428e51e555249e0085058e47c7736bf912815a80 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Mon, 28 Oct 2024 17:57:13 +0000 Subject: [PATCH 1/5] Invert structure of generated modules, with name and version at the top level --- src/deploy_tools/apptainer.py | 12 +++++------ src/deploy_tools/command.py | 6 +++--- src/deploy_tools/layout.py | 28 +++++++++++++++---------- src/deploy_tools/module.py | 19 +++++++++-------- src/deploy_tools/remove.py | 7 +++---- src/deploy_tools/remove_name_folders.py | 5 +---- src/deploy_tools/shell.py | 6 +++--- 7 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/deploy_tools/apptainer.py b/src/deploy_tools/apptainer.py index da95775..ca96c18 100644 --- a/src/deploy_tools/apptainer.py +++ b/src/deploy_tools/apptainer.py @@ -18,14 +18,14 @@ class ApptainerCreator: def __init__(self, templater: Templater, layout: Layout) -> None: self._templater = templater - self._entrypoints_root = layout.entrypoints_root - self._sif_root = layout.sif_files_root + self._layout = layout def create_entrypoint_files(self, config: Apptainer, module: Module) -> None: self._generate_sif_file(config, module) + metadata = module.metadata - entrypoints_folder = ( - self._entrypoints_root / module.metadata.name / module.metadata.version + entrypoints_folder = self._layout.get_entrypoints_folder( + metadata.name, metadata.version ) entrypoints_folder.mkdir(parents=True, exist_ok=True) sif_file = self._get_sif_file_path(config, module.metadata) @@ -72,6 +72,6 @@ def _generate_sif_file(self, config: Apptainer, module: Module) -> None: subprocess.run(commands, check=True) def _get_sif_file_path(self, config: Apptainer, metadata: ModuleMetadata) -> Path: - sif_parent = self._sif_root / metadata.name / metadata.version + sif_folder = self._layout.get_sif_files_folder(metadata.name, metadata.version) file_name = uuid.uuid3(uuid.NAMESPACE_URL, config.container.url).hex - return sif_parent / f"{file_name}.sif" + return sif_folder / f"{file_name}.sif" diff --git a/src/deploy_tools/command.py b/src/deploy_tools/command.py index 60d46da..2a0393e 100644 --- a/src/deploy_tools/command.py +++ b/src/deploy_tools/command.py @@ -9,15 +9,15 @@ class CommandCreator: def __init__(self, templater: Templater, layout: Layout) -> None: self._templater = templater - self._entrypoints_root = layout.entrypoints_root + self._layout = layout def create_entrypoint_file( self, config: Command, module: Module, ) -> None: - entrypoints_folder = ( - self._entrypoints_root / module.metadata.name / module.metadata.version + entrypoints_folder = self._layout.get_entrypoints_folder( + module.metadata.name, module.metadata.version ) entrypoints_folder.mkdir(parents=True, exist_ok=True) entrypoint_file = entrypoints_folder / config.name diff --git a/src/deploy_tools/layout.py b/src/deploy_tools/layout.py index 0966bb6..75fec5a 100644 --- a/src/deploy_tools/layout.py +++ b/src/deploy_tools/layout.py @@ -4,16 +4,29 @@ class Layout: """Represents the layout of the deployment area.""" - ENTRYPOINTS_ROOT_NAME = "entrypoints" + MODULES_ROOT_NAME = "modules" MODULEFILES_ROOT_NAME = "modulefiles" - SIF_FILES_ROOT_NAME = "sif_files" DEPRECATED_ROOT_NAME = "deprecated" + ENTRYPOINTS_FOLDER = "entrypoints" + SIF_FILES_FOLDER = "sif_files" DEPLOYMENT_SNAPSHOT_FILENAME = "deployment.yaml" def __init__(self, deployment_root: Path) -> None: self._root = deployment_root + def get_entrypoints_folder(self, name: str, version: str): + return self.modules_root / name / version / self.ENTRYPOINTS_FOLDER + + def get_sif_files_folder(self, name: str, version: str): + return self.modules_root / name / version / self.SIF_FILES_FOLDER + + def get_application_paths(self, name: str, version: str) -> list[Path]: + return [ + self.get_entrypoints_folder(name, version), + self.get_sif_files_folder(name, version), + ] + @property def deployment_root(self) -> Path: return self._root @@ -23,12 +36,8 @@ def deprecated_root(self) -> Path: return self._root / self.DEPRECATED_ROOT_NAME @property - def entrypoints_root(self) -> Path: - return self._root / self.ENTRYPOINTS_ROOT_NAME - - @property - def sif_files_root(self) -> Path: - return self._root / self.SIF_FILES_ROOT_NAME + def modules_root(self) -> Path: + return self._root / self.MODULES_ROOT_NAME @property def modulefiles_root(self) -> Path: @@ -41,6 +50,3 @@ def deprecated_modulefiles_root(self) -> Path: @property def snapshot_file(self) -> Path: return self._root / self.DEPLOYMENT_SNAPSHOT_FILENAME - - def get_application_paths(self) -> list[Path]: - return [self.entrypoints_root, self.sif_files_root] diff --git a/src/deploy_tools/module.py b/src/deploy_tools/module.py index bb6ec56..d3990f6 100644 --- a/src/deploy_tools/module.py +++ b/src/deploy_tools/module.py @@ -21,25 +21,26 @@ def __init__(self, templater: Templater, layout: Layout) -> None: self._templater = templater self._layout = layout self._modulefiles_root = layout.modulefiles_root - self._entrypoints_root = layout.entrypoints_root def create_module_file(self, module: Module) -> None: - config = module.metadata - entrypoints_folder = self._entrypoints_root / config.name / config.version + metadata = module.metadata + entrypoints_folder = self._layout.get_entrypoints_folder( + metadata.name, metadata.version + ) - description = config.description + description = metadata.description if description is None: - description = f"Scripts for {config.name}" + description = f"Scripts for {metadata.name}" params = { - "module_name": config.name, + "module_name": metadata.name, "module_description": description, - "env_vars": config.env_vars, - "dependencies": config.dependencies, + "env_vars": metadata.env_vars, + "dependencies": metadata.dependencies, "entrypoint_folder": entrypoints_folder, } - module_file = self._modulefiles_root / config.name / config.version + module_file = self._modulefiles_root / metadata.name / metadata.version module_file.parent.mkdir(exist_ok=True, parents=True) self._templater.create(module_file, TemplateType.MODULEFILE, params) diff --git a/src/deploy_tools/remove.py b/src/deploy_tools/remove.py index b22fe98..7065ec2 100644 --- a/src/deploy_tools/remove.py +++ b/src/deploy_tools/remove.py @@ -54,8 +54,7 @@ def remove_deprecated_module(name: str, version: str, layout: Layout) -> None: def remove_application_paths(name: str, version: str, layout: Layout) -> None: - to_remove = layout.get_application_paths() + to_remove = layout.get_application_paths(name, version) for path in to_remove: - version_path = path / name / version - if version_path.exists(): - shutil.rmtree(version_path) + if path.exists(): + shutil.rmtree(path) diff --git a/src/deploy_tools/remove_name_folders.py b/src/deploy_tools/remove_name_folders.py index 22e3faf..122f1e4 100644 --- a/src/deploy_tools/remove_name_folders.py +++ b/src/deploy_tools/remove_name_folders.py @@ -21,12 +21,9 @@ def remove_name_folders( for module in restored: delete_name_folder(module.metadata.name, layout.deprecated_modulefiles_root) - app_roots = layout.get_application_paths() for module in removed: delete_name_folder(module.metadata.name, layout.deprecated_modulefiles_root) - - for root in app_roots: - delete_name_folder(module.metadata.name, root) + delete_name_folder(module.metadata.name, layout.modules_root) def delete_name_folder(name: str, area_root: Path) -> None: diff --git a/src/deploy_tools/shell.py b/src/deploy_tools/shell.py index b5c426c..e574e20 100644 --- a/src/deploy_tools/shell.py +++ b/src/deploy_tools/shell.py @@ -12,15 +12,15 @@ class ShellCreator: def __init__(self, templater: Templater, layout: Layout) -> None: self._templater = templater - self._entrypoints_root = layout.entrypoints_root + self._layout = layout def create_entrypoint_file( self, config: Shell, module: Module, ) -> None: - entrypoints_folder = ( - self._entrypoints_root / module.metadata.name / module.metadata.version + entrypoints_folder = self._layout.get_entrypoints_folder( + module.metadata.name, module.metadata.version ) entrypoints_folder.mkdir(parents=True, exist_ok=True) entrypoint_file = entrypoints_folder / config.name From e5cece0fa746d5516631d106ee38c3cefff24874 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 29 Oct 2024 13:13:28 +0000 Subject: [PATCH 2/5] Correctly remove version from defaults list when module deprecated --- src/deploy_tools/validate.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index afabde1..b4abebc 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -230,12 +230,15 @@ def get_final_deployed_module_versions( """Return module versions that will exist after sync action has been carried out.""" final_versions: ModuleVersionsByName = defaultdict(list) for name, module_versions in deployment.modules.items(): - final_versions[name] = [ + versions = [ version for version, module in module_versions.items() if not module.metadata.deprecated ] + if versions: + final_versions[name] = versions + return final_versions From ecf6e164038f232ca1922be4cc125c58be8abd7f Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 29 Oct 2024 13:14:28 +0000 Subject: [PATCH 3/5] Fix removal of modules with new inverted structure --- src/deploy_tools/layout.py | 13 +++++-------- src/deploy_tools/remove.py | 13 ++++++------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/deploy_tools/layout.py b/src/deploy_tools/layout.py index 75fec5a..e48a61a 100644 --- a/src/deploy_tools/layout.py +++ b/src/deploy_tools/layout.py @@ -15,17 +15,14 @@ class Layout: def __init__(self, deployment_root: Path) -> None: self._root = deployment_root + def get_module_folder(self, name: str, version: str) -> Path: + return self.modules_root / name / version + def get_entrypoints_folder(self, name: str, version: str): - return self.modules_root / name / version / self.ENTRYPOINTS_FOLDER + return self.get_module_folder(name, version) / self.ENTRYPOINTS_FOLDER def get_sif_files_folder(self, name: str, version: str): - return self.modules_root / name / version / self.SIF_FILES_FOLDER - - def get_application_paths(self, name: str, version: str) -> list[Path]: - return [ - self.get_entrypoints_folder(name, version), - self.get_sif_files_folder(name, version), - ] + return self.get_module_folder(name, version) / self.SIF_FILES_FOLDER @property def deployment_root(self) -> Path: diff --git a/src/deploy_tools/remove.py b/src/deploy_tools/remove.py index 7065ec2..a60edd0 100644 --- a/src/deploy_tools/remove.py +++ b/src/deploy_tools/remove.py @@ -43,18 +43,17 @@ def remove_deployed_module(name: str, version: str, layout: Layout) -> None: module_file = layout.modulefiles_root / name / version module_file.unlink() - remove_application_paths(name, version, layout) + remove_module(name, version, layout) def remove_deprecated_module(name: str, version: str, layout: Layout) -> None: module_file = layout.deprecated_modulefiles_root / name / version module_file.unlink() - remove_application_paths(name, version, layout) + remove_module(name, version, layout) -def remove_application_paths(name: str, version: str, layout: Layout) -> None: - to_remove = layout.get_application_paths(name, version) - for path in to_remove: - if path.exists(): - shutil.rmtree(path) +def remove_module(name: str, version: str, layout: Layout) -> None: + module_folder = layout.get_module_folder(name, version) + if module_folder.exists(): + shutil.rmtree(module_folder) From bfa0b6a99fde928c3a889802655787cfe58c11b9 Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 29 Oct 2024 13:15:22 +0000 Subject: [PATCH 4/5] Fix removal of name folders for modulefiles --- src/deploy_tools/module.py | 3 ++- src/deploy_tools/remove_name_folders.py | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/deploy_tools/module.py b/src/deploy_tools/module.py index d3990f6..96bc7ba 100644 --- a/src/deploy_tools/module.py +++ b/src/deploy_tools/module.py @@ -11,6 +11,7 @@ ModuleVersionsByName: TypeAlias = dict[str, list[str]] VERSION_FILENAME = ".version" +VERSION_GLOB = "*/[!.version]*" DEVELOPMENT_VERSION = "dev" @@ -85,7 +86,7 @@ def get_deployed_module_versions( ) found_modules: ModuleVersionsByName = defaultdict(list) - for version_path in modulefiles_root.glob("*/[!.version]*"): + for version_path in modulefiles_root.glob(VERSION_GLOB): found_modules[version_path.parent.name].append(version_path.name) return found_modules diff --git a/src/deploy_tools/remove_name_folders.py b/src/deploy_tools/remove_name_folders.py index 122f1e4..696438a 100644 --- a/src/deploy_tools/remove_name_folders.py +++ b/src/deploy_tools/remove_name_folders.py @@ -1,7 +1,9 @@ +import shutil from pathlib import Path from .layout import Layout from .models.module import Module +from .module import VERSION_GLOB class RemoveNameFoldersError(Exception): @@ -16,16 +18,26 @@ def remove_name_folders( ) -> None: """Remove module name folders where all versions have been removed.""" for module in deprecated: - delete_name_folder(module.metadata.name, layout.modulefiles_root) + delete_modulefile_name_folder(module.metadata.name, layout.modulefiles_root) for module in restored: - delete_name_folder(module.metadata.name, layout.deprecated_modulefiles_root) + delete_modulefile_name_folder( + module.metadata.name, layout.deprecated_modulefiles_root + ) for module in removed: - delete_name_folder(module.metadata.name, layout.deprecated_modulefiles_root) + delete_modulefile_name_folder( + module.metadata.name, layout.deprecated_modulefiles_root + ) delete_name_folder(module.metadata.name, layout.modules_root) +def delete_modulefile_name_folder(name: str, modulefiles_root: Path) -> None: + modulefiles_name_path = modulefiles_root / name + if modulefiles_name_path.glob(VERSION_GLOB): + shutil.rmtree(modulefiles_name_path) + + def delete_name_folder(name: str, area_root: Path) -> None: name_folder = area_root / name try: From 2f458160a7d17927107e79589bd53404e479b52c Mon Sep 17 00:00:00 2001 From: Martin Gaughran Date: Tue, 29 Oct 2024 13:25:05 +0000 Subject: [PATCH 5/5] Rename validate_deployment to validate_update_group for consistency --- src/deploy_tools/sync.py | 4 ++-- src/deploy_tools/validate.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/deploy_tools/sync.py b/src/deploy_tools/sync.py index bca7a35..002f630 100644 --- a/src/deploy_tools/sync.py +++ b/src/deploy_tools/sync.py @@ -15,7 +15,7 @@ UpdateGroup, check_actions, validate_default_versions, - validate_deployment, + validate_update_group, ) @@ -25,7 +25,7 @@ def synchronise(deployment_root: Path, config_folder: Path) -> None: layout = Layout(deployment_root) snapshot = load_snapshot(layout) - update_group = validate_deployment(deployment, snapshot) + update_group = validate_update_group(deployment, snapshot) default_versions = validate_default_versions(deployment) check_actions(update_group, default_versions, layout) diff --git a/src/deploy_tools/validate.py b/src/deploy_tools/validate.py index b4abebc..3af162f 100644 --- a/src/deploy_tools/validate.py +++ b/src/deploy_tools/validate.py @@ -49,7 +49,7 @@ def validate_configuration(deployment_root: Path, config_folder: Path) -> None: layout = Layout(deployment_root) snapshot = load_snapshot(layout) - update_group = validate_deployment(deployment, snapshot) + update_group = validate_update_group(deployment, snapshot) default_versions = validate_default_versions(deployment) check_actions(update_group, default_versions, layout) @@ -106,7 +106,7 @@ def print_version_updates( print() -def validate_deployment(deployment: Deployment, snapshot: Deployment) -> UpdateGroup: +def validate_update_group(deployment: Deployment, snapshot: Deployment) -> UpdateGroup: """Validate configuration to get set of actions that need to be carried out.""" old_modules = snapshot.modules new_modules = deployment.modules