diff --git a/src/deploy_tools/app_builder.py b/src/deploy_tools/app_builder.py index f4b019a..6a62fd3 100644 --- a/src/deploy_tools/app_builder.py +++ b/src/deploy_tools/app_builder.py @@ -1,6 +1,10 @@ +import hashlib import uuid from itertools import chain from pathlib import Path +from urllib.request import urlretrieve + +from deploy_tools.models.binary_app import BinaryApp, HashType from .apptainer import create_sif_file from .layout import ModuleBuildLayout @@ -9,6 +13,8 @@ from .models.shell_app import ShellApp from .templater import Templater, TemplateType +ALL_READ_EXECUTE_PERMISSIONS = 0o555 + class AppBuilderError(Exception): pass @@ -27,6 +33,8 @@ def create_application_files(self, app: Application, module: Module): self._create_apptainer_files(app, module) case ShellApp(): self._create_shell_file(app, module) + case BinaryApp(): + self._create_binary_file(app, module) def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None: """Create apptainer entrypoints using a specified image and commands.""" @@ -100,3 +108,37 @@ def _create_shell_file(self, app: ShellApp, module: Module) -> None: executable=True, create_parents=True, ) + + def _create_binary_file(self, app: BinaryApp, module: Module) -> None: + """ + Download a URL, validate it against its hash, make it executable + and add it to PATH + """ + binary_folder = self._build_layout.get_entrypoints_folder( + module.name, module.version + ) + binary_path = binary_folder / app.name + binary_path.parent.mkdir(parents=True, exist_ok=True) + urlretrieve(app.url, binary_path) + + match app.hash_type: + case HashType.SHA256: + h = hashlib.sha256() + case HashType.SHA512: + h = hashlib.sha512() + case HashType.MD5: + h = hashlib.md5() + case HashType.NONE: + h = None + + if h is not None: + with open(binary_path, "rb") as fh: + while True: + data = fh.read(4096) + if len(data) == 0: + break + h.update(data) + if h.hexdigest() != app.hash: + raise AppBuilderError(f"Downloaded Binary {app.url} hash check failed") + + binary_path.chmod(ALL_READ_EXECUTE_PERMISSIONS) diff --git a/src/deploy_tools/demo_configuration/argocd/0.1.yaml b/src/deploy_tools/demo_configuration/argocd/0.1.yaml new file mode 100644 index 0000000..d5497bc --- /dev/null +++ b/src/deploy_tools/demo_configuration/argocd/0.1.yaml @@ -0,0 +1,13 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: argocd + version: "0.1" + description: Demonstration of binary download + + applications: + - app_type: binary + name: argocd + url: https://github.com/argoproj/argo-cd/releases/download/v2.14.10/argocd-linux-amd64 + hash_type: sha256 + hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa diff --git a/src/deploy_tools/models/binary_app.py b/src/deploy_tools/models/binary_app.py new file mode 100644 index 0000000..e283164 --- /dev/null +++ b/src/deploy_tools/models/binary_app.py @@ -0,0 +1,42 @@ +from enum import StrEnum +from typing import Literal + +from pydantic import Field + +from .parent import ParentModel + + +class HashType(StrEnum): + """Type of hash to use for the binary.""" + + SHA256 = "sha256" + SHA512 = "sha512" + MD5 = "md5" + NONE = "none" + + +class BinaryApp(ParentModel): + """ + Represents a standalone Binary application. + + This will fetch a standalone binary, validate its hash and add its + location to that path. + """ + + app_type: Literal["binary"] + name: str = Field( + ..., + description="Binary filename to use locally", + ) + url: str = Field( + ..., + description="URL to download the binary from.", + ) + hash: str = Field( + "", + description="Hash to verify binary integrity", + ) + hash_type: HashType = Field( + ..., + description="Type of hash used to check the binary.", + ) diff --git a/src/deploy_tools/models/module.py b/src/deploy_tools/models/module.py index eaa4acd..6ff963b 100644 --- a/src/deploy_tools/models/module.py +++ b/src/deploy_tools/models/module.py @@ -3,11 +3,15 @@ from pydantic import Field +from deploy_tools.models.binary_app import BinaryApp + from .apptainer_app import ApptainerApp from .parent import ParentModel from .shell_app import ShellApp -Application = Annotated[ApptainerApp | ShellApp, Field(..., discriminator="app_type")] +Application = Annotated[ + ApptainerApp | ShellApp | BinaryApp, Field(..., discriminator="app_type") +] DEVELOPMENT_VERSION = "dev" diff --git a/src/deploy_tools/models/parent.py b/src/deploy_tools/models/parent.py index 2e50d19..ee1ca82 100644 --- a/src/deploy_tools/models/parent.py +++ b/src/deploy_tools/models/parent.py @@ -2,6 +2,15 @@ class ParentModel(BaseModel): - """Will forbid any extra parameters being provided in any subclass.""" + """ + Provides Model Config for all Pydantic models in this project: - model_config = ConfigDict(extra="forbid") + forbid: forbid any extra parameters being provided in any subclass. + use_enum_values: use the enum value only when serializing the model, + this means the yaml serializer can work with enums + """ + + model_config = ConfigDict( + extra="forbid", + use_enum_values=True, + ) diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index 1746d19..96be029 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -37,6 +37,45 @@ "title": "ApptainerApp", "type": "object" }, + "BinaryApp": { + "additionalProperties": false, + "description": "Represents a standalone Binary application.\n\nThis will fetch a standalone binary, validate its hash and add its\nlocation to that path.", + "properties": { + "app_type": { + "const": "binary", + "title": "App Type", + "type": "string" + }, + "name": { + "description": "Binary filename to use locally", + "title": "Name", + "type": "string" + }, + "url": { + "description": "URL to download the binary from.", + "title": "Url", + "type": "string" + }, + "hash": { + "default": "", + "description": "Hash to verify binary integrity", + "title": "Hash", + "type": "string" + }, + "hash_type": { + "$ref": "#/$defs/HashType", + "description": "Type of hash used to check the binary." + } + }, + "required": [ + "app_type", + "name", + "url", + "hash_type" + ], + "title": "BinaryApp", + "type": "object" + }, "ContainerImage": { "additionalProperties": false, "properties": { @@ -167,6 +206,17 @@ "title": "EnvVar", "type": "object" }, + "HashType": { + "description": "Type of hash to use for the binary.", + "enum": [ + "sha256", + "sha512", + "md5", + "none" + ], + "title": "HashType", + "type": "string" + }, "Module": { "additionalProperties": false, "description": "Represents a Module to be deployed.\n\nModules can optionally include a set of applications, environment variables to load,\nand a list of module dependencies.", @@ -212,6 +262,7 @@ "discriminator": { "mapping": { "apptainer": "#/$defs/ApptainerApp", + "binary": "#/$defs/BinaryApp", "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" @@ -222,6 +273,9 @@ }, { "$ref": "#/$defs/ShellApp" + }, + { + "$ref": "#/$defs/BinaryApp" } ] }, diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index 8007d8b..e7488fe 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -37,6 +37,45 @@ "title": "ApptainerApp", "type": "object" }, + "BinaryApp": { + "additionalProperties": false, + "description": "Represents a standalone Binary application.\n\nThis will fetch a standalone binary, validate its hash and add its\nlocation to that path.", + "properties": { + "app_type": { + "const": "binary", + "title": "App Type", + "type": "string" + }, + "name": { + "description": "Binary filename to use locally", + "title": "Name", + "type": "string" + }, + "url": { + "description": "URL to download the binary from.", + "title": "Url", + "type": "string" + }, + "hash": { + "default": "", + "description": "Hash to verify binary integrity", + "title": "Hash", + "type": "string" + }, + "hash_type": { + "$ref": "#/$defs/HashType", + "description": "Type of hash used to check the binary." + } + }, + "required": [ + "app_type", + "name", + "url", + "hash_type" + ], + "title": "BinaryApp", + "type": "object" + }, "ContainerImage": { "additionalProperties": false, "properties": { @@ -149,6 +188,17 @@ "title": "EnvVar", "type": "object" }, + "HashType": { + "description": "Type of hash to use for the binary.", + "enum": [ + "sha256", + "sha512", + "md5", + "none" + ], + "title": "HashType", + "type": "string" + }, "Module": { "additionalProperties": false, "description": "Represents a Module to be deployed.\n\nModules can optionally include a set of applications, environment variables to load,\nand a list of module dependencies.", @@ -194,6 +244,7 @@ "discriminator": { "mapping": { "apptainer": "#/$defs/ApptainerApp", + "binary": "#/$defs/BinaryApp", "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" @@ -204,6 +255,9 @@ }, { "$ref": "#/$defs/ShellApp" + }, + { + "$ref": "#/$defs/BinaryApp" } ] }, diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index 8007d8b..e7488fe 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -37,6 +37,45 @@ "title": "ApptainerApp", "type": "object" }, + "BinaryApp": { + "additionalProperties": false, + "description": "Represents a standalone Binary application.\n\nThis will fetch a standalone binary, validate its hash and add its\nlocation to that path.", + "properties": { + "app_type": { + "const": "binary", + "title": "App Type", + "type": "string" + }, + "name": { + "description": "Binary filename to use locally", + "title": "Name", + "type": "string" + }, + "url": { + "description": "URL to download the binary from.", + "title": "Url", + "type": "string" + }, + "hash": { + "default": "", + "description": "Hash to verify binary integrity", + "title": "Hash", + "type": "string" + }, + "hash_type": { + "$ref": "#/$defs/HashType", + "description": "Type of hash used to check the binary." + } + }, + "required": [ + "app_type", + "name", + "url", + "hash_type" + ], + "title": "BinaryApp", + "type": "object" + }, "ContainerImage": { "additionalProperties": false, "properties": { @@ -149,6 +188,17 @@ "title": "EnvVar", "type": "object" }, + "HashType": { + "description": "Type of hash to use for the binary.", + "enum": [ + "sha256", + "sha512", + "md5", + "none" + ], + "title": "HashType", + "type": "string" + }, "Module": { "additionalProperties": false, "description": "Represents a Module to be deployed.\n\nModules can optionally include a set of applications, environment variables to load,\nand a list of module dependencies.", @@ -194,6 +244,7 @@ "discriminator": { "mapping": { "apptainer": "#/$defs/ApptainerApp", + "binary": "#/$defs/BinaryApp", "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" @@ -204,6 +255,9 @@ }, { "$ref": "#/$defs/ShellApp" + }, + { + "$ref": "#/$defs/BinaryApp" } ] }, diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh index 736c713..c452420 100755 --- a/tests/generate_samples.sh +++ b/tests/generate_samples.sh @@ -12,14 +12,12 @@ mkdir -p "${TMP_DIR}" deploy-tools sync --from-scratch ${TMP_DIR} ${THIS_DIR}/../src/deploy_tools/demo_configuration -# don't keep the sif or git files! +# don't keep the sif or git files rm -rf $(find ${TMP_DIR} -name "*.sif") rm -rf ${TMP_DIR}/.git* +# also remove binaries +rm ${TMP_DIR}/modules/argocd/0.1/entrypoints/argocd rm -rf ${SAMPLES_DIR} mkdir -p ${SAMPLES_DIR} cp -r ${TMP_DIR} ${SAMPLES_DIR} -<<<<<<< HEAD -======= - ->>>>>>> ce3367b (add test for demo_configuration) diff --git a/tests/samples/deploy-tools-output/deployment.yaml b/tests/samples/deploy-tools-output/deployment.yaml index ced3c36..122861a 100644 --- a/tests/samples/deploy-tools-output/deployment.yaml +++ b/tests/samples/deploy-tools-output/deployment.yaml @@ -1,4 +1,19 @@ releases: + argocd: + '0.1': + deprecated: false + module: + applications: + - app_type: binary + hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa + hash_type: sha256 + name: argocd + url: https://github.com/argoproj/argo-cd/releases/download/v2.14.10/argocd-linux-amd64 + dependencies: [] + description: Demonstration of binary download + env_vars: [] + name: argocd + version: '0.1' dls-pmac-control: '0.1': deprecated: false diff --git a/tests/samples/deploy-tools-output/modulefiles/argocd/.version b/tests/samples/deploy-tools-output/modulefiles/argocd/.version new file mode 100644 index 0000000..d9b8f92 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/argocd/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.1 diff --git a/tests/samples/deploy-tools-output/modulefiles/argocd/0.1 b/tests/samples/deploy-tools-output/modulefiles/argocd/0.1 new file mode 120000 index 0000000..28088dc --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/argocd/0.1 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/argocd/0.1/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/argocd/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/argocd/0.1/module.yaml new file mode 100644 index 0000000..6f72dc8 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/argocd/0.1/module.yaml @@ -0,0 +1,11 @@ +applications: +- app_type: binary + hash: d1750274a336f0a090abf196a832cee14cb9f1c2fc3d20d80b0dbfeff83550fa + hash_type: sha256 + name: argocd + url: https://github.com/argoproj/argo-cd/releases/download/v2.14.10/argocd-linux-amd64 +dependencies: [] +description: Demonstration of binary download +env_vars: [] +name: argocd +version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/argocd/0.1/modulefile b/tests/samples/deploy-tools-output/modules/argocd/0.1/modulefile new file mode 100644 index 0000000..c89943d --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/argocd/0.1/modulefile @@ -0,0 +1,9 @@ +#%Module1.0 +## +## argocd - Demonstration of binary download +## +module-whatis "Demonstration of binary download" + + + +prepend-path PATH "/tmp/deploy-tools-output/modules/argocd/0.1/entrypoints" diff --git a/tests/test_cli.py b/tests/test_cli.py index 31e378a..df8efb7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,6 +7,10 @@ from conftest import run_cli from deploy_tools import __version__ +PATH_TO_SCHEMAS = ( + Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" +) + def test_cli_version(): cmd = [sys.executable, "-m", "deploy_tools", "--version"]