From d3334c9be1f9163c6800035b69870a2a2ea4686d Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 05:34:13 +0000 Subject: [PATCH 01/12] add schema checking test --- .devcontainer/devcontainer.json | 2 +- .gitignore | 3 +++ .../models/schemas/deployment.json | 16 ++++++++-------- src/deploy_tools/models/schemas/module.json | 16 ++++++++-------- src/deploy_tools/models/schemas/release.json | 16 ++++++++-------- tests/conftest.py | 13 +++++++++++++ tests/test_cli.py | 19 +++++++++++++++++++ 7 files changed, 60 insertions(+), 25 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 46a7e3d..ee851e8 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,7 @@ }, "features": { // add in eternal history and other bash features - "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1.0.0": {} + "ghcr.io/diamondlightsource/devcontainer-features/bash-config:1": {} }, // Create the config folder for the bash-config feature "initializeCommand": "mkdir -p ${localEnv:HOME}/.config/bash-config", diff --git a/.gitignore b/.gitignore index 5eda15c..c602ca4 100644 --- a/.gitignore +++ b/.gitignore @@ -72,3 +72,6 @@ lockfiles/ # Demo output path for vscode tasks demo-output + +# vscode workspaces are local only +*.code-workspace diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index 3ee93bc..47b047b 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -1,6 +1,6 @@ { "$defs": { - "Apptainer": { + "ApptainerApp": { "additionalProperties": false, "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.", "properties": { @@ -33,7 +33,7 @@ "container", "entrypoints" ], - "title": "Apptainer", + "title": "ApptainerApp", "type": "object" }, "ContainerImage": { @@ -197,17 +197,17 @@ "items": { "discriminator": { "mapping": { - "apptainer": "#/$defs/Apptainer", - "shell": "#/$defs/Shell" + "apptainer": "#/$defs/ApptainerApp", + "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" }, "oneOf": [ { - "$ref": "#/$defs/Apptainer" + "$ref": "#/$defs/ApptainerApp" }, { - "$ref": "#/$defs/Shell" + "$ref": "#/$defs/ShellApp" } ] }, @@ -281,7 +281,7 @@ }, "type": "object" }, - "Shell": { + "ShellApp": { "additionalProperties": false, "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.", "properties": { @@ -307,7 +307,7 @@ "name", "script" ], - "title": "Shell", + "title": "ShellApp", "type": "object" } }, diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index e7a877e..b1ea494 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -1,6 +1,6 @@ { "$defs": { - "Apptainer": { + "ApptainerApp": { "additionalProperties": false, "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.", "properties": { @@ -33,7 +33,7 @@ "container", "entrypoints" ], - "title": "Apptainer", + "title": "ApptainerApp", "type": "object" }, "ContainerImage": { @@ -179,17 +179,17 @@ "items": { "discriminator": { "mapping": { - "apptainer": "#/$defs/Apptainer", - "shell": "#/$defs/Shell" + "apptainer": "#/$defs/ApptainerApp", + "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" }, "oneOf": [ { - "$ref": "#/$defs/Apptainer" + "$ref": "#/$defs/ApptainerApp" }, { - "$ref": "#/$defs/Shell" + "$ref": "#/$defs/ShellApp" } ] }, @@ -232,7 +232,7 @@ "title": "ModuleDependency", "type": "object" }, - "Shell": { + "ShellApp": { "additionalProperties": false, "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.", "properties": { @@ -258,7 +258,7 @@ "name", "script" ], - "title": "Shell", + "title": "ShellApp", "type": "object" } }, diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index e7a877e..b1ea494 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -1,6 +1,6 @@ { "$defs": { - "Apptainer": { + "ApptainerApp": { "additionalProperties": false, "description": "Represents an Apptainer application or set of applications for a single image.\n\nThis uses Apptainer to deploy a portable image of the desired container. Several\nentrypoints can then be specified to allow for multiple commands run on the same\ncontainer image.", "properties": { @@ -33,7 +33,7 @@ "container", "entrypoints" ], - "title": "Apptainer", + "title": "ApptainerApp", "type": "object" }, "ContainerImage": { @@ -179,17 +179,17 @@ "items": { "discriminator": { "mapping": { - "apptainer": "#/$defs/Apptainer", - "shell": "#/$defs/Shell" + "apptainer": "#/$defs/ApptainerApp", + "shell": "#/$defs/ShellApp" }, "propertyName": "app_type" }, "oneOf": [ { - "$ref": "#/$defs/Apptainer" + "$ref": "#/$defs/ApptainerApp" }, { - "$ref": "#/$defs/Shell" + "$ref": "#/$defs/ShellApp" } ] }, @@ -232,7 +232,7 @@ "title": "ModuleDependency", "type": "object" }, - "Shell": { + "ShellApp": { "additionalProperties": false, "description": "Represents a Shell application.\n\nThis will run the code specified as a shell script. This currently uses Bash for\nimproved functionality while retaining high compatibility with various Linux\ndistributions.", "properties": { @@ -258,7 +258,7 @@ "name", "script" ], - "title": "Shell", + "title": "ShellApp", "type": "object" } }, diff --git a/tests/conftest.py b/tests/conftest.py index ebe9c10..f33be41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,19 @@ from typing import Any import pytest +from typer.testing import CliRunner + +from deploy_tools.__main__ import app + +runner = CliRunner() + + +def run_cli(*args): + result = runner.invoke(app, [str(x) for x in args]) + if result.exception: + raise result.exception + assert result.exit_code == 0, result + # Prevent pytest from catching exceptions when debugging in vscode so that break on # exception works correctly (see: https://github.com/pytest-dev/pytest/issues/7409) diff --git a/tests/test_cli.py b/tests/test_cli.py index 745afdd..368f8da 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,9 +1,28 @@ import subprocess import sys +import tempfile +from pathlib import Path +from conftest import run_cli from deploy_tools import __version__ +PATH_TO_SCHEMAS = ( + Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" +) + def test_cli_version(): cmd = [sys.executable, "-m", "deploy_tools", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ + + +def test_schema(): + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + # Generate up to date schema files + run_cli("schema", tmp_path) + # Compare with the expected schema files + for schema in tmp_path.glob("*.json"): + expected = PATH_TO_SCHEMAS / schema.name + if schema.read_text() != expected.read_text(): + raise AssertionError(f"Schema file {expected} is out of date.") From a4ba878a331d5f5e03b63a2bb00ade9b4a6f1fb1 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 06:49:06 +0000 Subject: [PATCH 02/12] add host_binaries feature --- .devcontainer/devcontainer.json | 6 ++- src/deploy_tools/app_builder.py | 4 ++ .../edge-containers-cli/0.1.yaml | 54 +++++++++++++++++++ src/deploy_tools/models/apptainer_app.py | 24 +++++++-- .../models/schemas/deployment.json | 18 ++++++- src/deploy_tools/models/schemas/module.json | 18 ++++++- src/deploy_tools/models/schemas/release.json | 18 ++++++- .../templates/apptainer_entrypoint | 11 +++- 8 files changed, 142 insertions(+), 11 deletions(-) create mode 100644 src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ee851e8..12d2f01 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -42,7 +42,11 @@ "--device=/dev/fuse" ], "mounts": [ - "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached" + "source=/dls_sw/,target=/dls_sw,type=bind,consistency=cached", + // these two mounts allow us to pick up cache and config folders that have + // been symlinked out of the home directory + "source=/dls/science/users/,target=/dls/science/users,type=bind,consistency=cached", + "source=/scratch/,target=/scratch,type=bind,consistency=cached" ], // Mount the parent as /workspaces so we can pip install peers as editable "workspaceMount": "source=${localWorkspaceFolder}/..,target=/workspaces,type=bind", diff --git a/src/deploy_tools/app_builder.py b/src/deploy_tools/app_builder.py index 7e4dc41..f4b019a 100644 --- a/src/deploy_tools/app_builder.py +++ b/src/deploy_tools/app_builder.py @@ -44,6 +44,9 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None: entrypoint_file = entrypoints_folder / entrypoint.name mounts = ",".join(chain(global_options.mounts, options.mounts)).strip() + host_binaries = " ".join( + chain(global_options.host_binaries, options.host_binaries) + ).strip() apptainer_args = f"{global_options.apptainer_args} {options.apptainer_args}" apptainer_args = apptainer_args.strip() @@ -55,6 +58,7 @@ def _create_apptainer_files(self, app: ApptainerApp, module: Module) -> None: params = { "mounts": mounts, + "host_binaries": host_binaries, "apptainer_args": apptainer_args, "relative_sif_file": relative_sif_file, "command": command, diff --git a/src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml b/src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml new file mode 100644 index 0000000..8dc38bb --- /dev/null +++ b/src/deploy_tools/demo_configuration/edge-containers-cli/0.1.yaml @@ -0,0 +1,54 @@ +# yaml-language-server: $schema=/workspaces/deploy-tools/src/deploy_tools/models/schemas/release.json + +module: + name: edge-containers-cli + version: "0.1" + description: ec command line tool for kubernetes IOCs + + env_vars: + # For host-binaries PR we will use the K8S backend - this will be changed to + # ARGOCD when we add in the BinaryApp feature (upcoming PR) + - name: EC_CLI_BACKEND + value: K8S + - name: EC_LOG_URL + value: "https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}*" + # The default configuration is for p47 for testing + - name: EC_TARGET + value: p47-beamline # K8S target + # value: p47-beamline/p47 # Argocd target + - name: EC_SERVICES_REPO + value: https://github.com/epics-containers/p47-services + + applications: + - app_type: apptainer + + container: + path: docker://ghcr.io/epics-containers/edge-containers-cli + version: "4.4.1" + + entrypoints: + - name: ec + command: ec + # for debugging enter the container with bash shell + - name: ec-bash + command: bash + + global_options: + mounts: + # places to get argocd and kubectl config from + - /dls/science/users/:/dls/science/users/ + - /scratch:/scratch + - /dls_sw/apps:/dls_sw/apps + host_binaries: + - kubectl + - helm + - kubelogin + - klogout + - kustomize + - kubeseal + - kubectl-oidc_login + + - app_type: shell + name: ec-login + script: + - argocd login argocd.diamond.ac.uk --grpc-web --sso diff --git a/src/deploy_tools/models/apptainer_app.py b/src/deploy_tools/models/apptainer_app.py index 35c6bf2..541fd16 100644 --- a/src/deploy_tools/models/apptainer_app.py +++ b/src/deploy_tools/models/apptainer_app.py @@ -1,13 +1,31 @@ from collections.abc import Sequence from typing import Literal +from pydantic import Field + from .parent import ParentModel class EntrypointOptions(ParentModel): - apptainer_args: str = "" - command_args: str = "" - mounts: Sequence[str] = [] + apptainer_args: str = Field( + description="Apptainer arguments to pass when launching the container", + default="", + ) + command_args: str = Field( + description="Arguments to pass to command entrypoint", + default="", + ) + mounts: Sequence[str] = Field( + description="A list of mount points to add to the container in the form of " + "'host_path:container_path'", + default=[], + ) + host_binaries: Sequence[str] = Field( + description="A list of host binaries to mount into the container. These are " + "discovered on the host using the current PATH and are mounted into the " + "container at /usr/bin/.", + default=[], + ) class Entrypoint(ParentModel): diff --git a/src/deploy_tools/models/schemas/deployment.json b/src/deploy_tools/models/schemas/deployment.json index 47b047b..1746d19 100644 --- a/src/deploy_tools/models/schemas/deployment.json +++ b/src/deploy_tools/models/schemas/deployment.json @@ -24,7 +24,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -98,7 +99,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -113,21 +115,33 @@ "properties": { "apptainer_args": { "default": "", + "description": "Apptainer arguments to pass when launching the container", "title": "Apptainer Args", "type": "string" }, "command_args": { "default": "", + "description": "Arguments to pass to command entrypoint", "title": "Command Args", "type": "string" }, "mounts": { "default": [], + "description": "A list of mount points to add to the container in the form of 'host_path:container_path'", "items": { "type": "string" }, "title": "Mounts", "type": "array" + }, + "host_binaries": { + "default": [], + "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/.", + "items": { + "type": "string" + }, + "title": "Host Binaries", + "type": "array" } }, "title": "EntrypointOptions", diff --git a/src/deploy_tools/models/schemas/module.json b/src/deploy_tools/models/schemas/module.json index b1ea494..8007d8b 100644 --- a/src/deploy_tools/models/schemas/module.json +++ b/src/deploy_tools/models/schemas/module.json @@ -24,7 +24,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -80,7 +81,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -95,21 +97,33 @@ "properties": { "apptainer_args": { "default": "", + "description": "Apptainer arguments to pass when launching the container", "title": "Apptainer Args", "type": "string" }, "command_args": { "default": "", + "description": "Arguments to pass to command entrypoint", "title": "Command Args", "type": "string" }, "mounts": { "default": [], + "description": "A list of mount points to add to the container in the form of 'host_path:container_path'", "items": { "type": "string" }, "title": "Mounts", "type": "array" + }, + "host_binaries": { + "default": [], + "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/.", + "items": { + "type": "string" + }, + "title": "Host Binaries", + "type": "array" } }, "title": "EntrypointOptions", diff --git a/src/deploy_tools/models/schemas/release.json b/src/deploy_tools/models/schemas/release.json index b1ea494..8007d8b 100644 --- a/src/deploy_tools/models/schemas/release.json +++ b/src/deploy_tools/models/schemas/release.json @@ -24,7 +24,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -80,7 +81,8 @@ "default": { "apptainer_args": "", "command_args": "", - "mounts": [] + "mounts": [], + "host_binaries": [] } } }, @@ -95,21 +97,33 @@ "properties": { "apptainer_args": { "default": "", + "description": "Apptainer arguments to pass when launching the container", "title": "Apptainer Args", "type": "string" }, "command_args": { "default": "", + "description": "Arguments to pass to command entrypoint", "title": "Command Args", "type": "string" }, "mounts": { "default": [], + "description": "A list of mount points to add to the container in the form of 'host_path:container_path'", "items": { "type": "string" }, "title": "Mounts", "type": "array" + }, + "host_binaries": { + "default": [], + "description": "A list of host binaries to mount into the container. These are discovered on the host using the current PATH and are mounted into the container at /usr/bin/.", + "items": { + "type": "string" + }, + "title": "Host Binaries", + "type": "array" } }, "title": "EntrypointOptions", diff --git a/src/deploy_tools/templates/apptainer_entrypoint b/src/deploy_tools/templates/apptainer_entrypoint index da9a834..bb9d350 100644 --- a/src/deploy_tools/templates/apptainer_entrypoint +++ b/src/deploy_tools/templates/apptainer_entrypoint @@ -23,6 +23,12 @@ if [[ ! -f ${sif_file} ]]; then exit 1 fi +# add mounts of host binaries into /usr/bin +for i in {{ host_binaries }}; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + # Set up mounts if any have been configured if [[ ! -z ${mounts} ]]; then opts="-B ${mounts}" @@ -31,5 +37,8 @@ fi opts=${opts}" --env DISPLAY=${DISPLAY}" opts=${opts}" ${apptainer_args}" -set -x + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" From 2d3c8bafb5edc50732cc49831638e9d7cfea339e Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 07:27:34 +0000 Subject: [PATCH 03/12] add test for demo_configuration --- tests/conftest.py | 11 + tests/generate_samples.sh | 22 ++ .../deploy-tools-output/deployment.yaml | 213 ++++++++++++++++++ .../modulefiles/dls-pmac-control/.version | 2 + .../modulefiles/dls-pmac-control/0.1 | 1 + .../modulefiles/dls-pmac-control/0.2 | 1 + .../modulefiles/edge-containers-cli/.version | 2 + .../modulefiles/edge-containers-cli/0.1 | 1 + .../modulefiles/example-module-apps/.version | 2 + .../modulefiles/example-module-apps/0.1 | 1 + .../modulefiles/example-module-deps/.version | 2 + .../modulefiles/example-module-deps/0.2 | 1 + .../modulefiles/phoebus/.version | 2 + .../modulefiles/phoebus/0.1 | 1 + .../0.1/entrypoints/dls-pmac-control | 44 ++++ .../0.1/entrypoints/test-echo-module-file | 3 + .../modules/dls-pmac-control/0.1/module.yaml | 29 +++ .../modules/dls-pmac-control/0.1/modulefile | 10 + .../0.2/entrypoints/dls-pmac-control | 44 ++++ .../0.2/entrypoints/test-echo-module-file | 3 + .../modules/dls-pmac-control/0.2/module.yaml | 29 +++ .../modules/dls-pmac-control/0.2/modulefile | 10 + .../edge-containers-cli/0.1/entrypoints/ec | 44 ++++ .../0.1/entrypoints/ec-bash | 44 ++++ .../0.1/entrypoints/ec-login | 3 + .../edge-containers-cli/0.1/module.yaml | 52 +++++ .../edge-containers-cli/0.1/modulefile | 13 ++ .../0.1/entrypoints/cowsay-hello | 44 ++++ .../0.1/entrypoints/show-directory | 44 ++++ .../0.1/entrypoints/test-echo-module-folder | 3 + .../0.1/entrypoints/test-shell-script | 5 + .../example-module-apps/0.1/module.yaml | 43 ++++ .../example-module-apps/0.1/modulefile | 10 + .../example-module-deps/0.2/module.yaml | 10 + .../example-module-deps/0.2/modulefile | 11 + .../modules/phoebus/0.1/entrypoints/phoebus | 44 ++++ .../modules/phoebus/0.1/module.yaml | 23 ++ .../modules/phoebus/0.1/modulefile | 9 + tests/test_cli.py | 30 +++ 39 files changed, 866 insertions(+) create mode 100755 tests/generate_samples.sh create mode 100644 tests/samples/deploy-tools-output/deployment.yaml create mode 100644 tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version create mode 120000 tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.1 create mode 120000 tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.2 create mode 100644 tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version create mode 120000 tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/0.1 create mode 100644 tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version create mode 120000 tests/samples/deploy-tools-output/modulefiles/example-module-apps/0.1 create mode 100644 tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version create mode 120000 tests/samples/deploy-tools-output/modulefiles/example-module-deps/0.2 create mode 100644 tests/samples/deploy-tools-output/modulefiles/phoebus/.version create mode 120000 tests/samples/deploy-tools-output/modulefiles/phoebus/0.1 create mode 100755 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control create mode 100755 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file create mode 100644 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile create mode 100755 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control create mode 100755 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file create mode 100644 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile create mode 100755 tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec create mode 100755 tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash create mode 100755 tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login create mode 100644 tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile create mode 100755 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello create mode 100755 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory create mode 100755 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder create mode 100755 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script create mode 100644 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile create mode 100644 tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile create mode 100755 tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus create mode 100644 tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml create mode 100644 tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile diff --git a/tests/conftest.py b/tests/conftest.py index f33be41..e44d8ce 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +from pathlib import Path from typing import Any import pytest @@ -32,3 +33,13 @@ def pytest_exception_interact(call: pytest.CallInfo[Any]): @pytest.hookimpl(tryfirst=True) def pytest_internalerror(excinfo: pytest.ExceptionInfo[Any]): raise excinfo.value + + +@pytest.fixture +def samples(): + return Path(__file__).parent / "samples" + + +@pytest.fixture +def demo_config(): + return Path(__file__).parent.parent / "src" / "deploy_tools" / "demo_configuration" diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh new file mode 100755 index 0000000..5ab5832 --- /dev/null +++ b/tests/generate_samples.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -xe + +THIS_DIR=$(realpath $(dirname ${0})) +SAMPLES_DIR=${THIS_DIR}/samples +TMP_DIR=/tmp/deploy-tools-output + +rm -rf "${SAMPLES_DIR}" +rm -rf "${TMP_DIR}" +mkdir -p "${TMP_DIR}" + +deploy-tools sync --from-scratch ${TMP_DIR} ${THIS_DIR}/../src/deploy_tools/demo_configuration + +# don't keep the sif or git files! +rm -rf $(find ${TMP_DIR} -name "*.sif") +rm -rf ${TMP_DIR}/.git* + +rm -rf ${SAMPLES_DIR} +mkdir -p ${SAMPLES_DIR} +cp -r ${TMP_DIR} ${SAMPLES_DIR} + diff --git a/tests/samples/deploy-tools-output/deployment.yaml b/tests/samples/deploy-tools-output/deployment.yaml new file mode 100644 index 0000000..ced3c36 --- /dev/null +++ b/tests/samples/deploy-tools-output/deployment.yaml @@ -0,0 +1,213 @@ +releases: + dls-pmac-control: + '0.1': + deprecated: false + module: + applications: + - app_type: apptainer + container: + path: docker://ghcr.io/diamondlightsource/dls-pmac-control + version: 3.2.0b1 + entrypoints: + - command: dls-pmac-control + name: dls-pmac-control + options: + apptainer_args: -e + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + - app_type: shell + name: test-echo-module-file + script: + - echo $EXAMPLE_VALUE + dependencies: [] + description: Demonstration of the deploy-tools process + env_vars: + - name: EXAMPLE_VALUE + value: Test message EXAMPLE_VALUE from example-module-file version 0.1 + name: dls-pmac-control + version: '0.1' + '0.2': + deprecated: false + module: + applications: + - app_type: apptainer + container: + path: docker://ghcr.io/diamondlightsource/dls-pmac-control + version: 3.2.0b1 + entrypoints: + - command: dls-pmac-control + name: dls-pmac-control + options: + apptainer_args: -e + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + - app_type: shell + name: test-echo-module-file + script: + - echo $EXAMPLE_VALUE + dependencies: [] + description: Demonstration of the deploy-tools process + env_vars: + - name: EXAMPLE_VALUE + value: Test message EXAMPLE_VALUE from example-module-file version 0.2 + name: dls-pmac-control + version: '0.2' + edge-containers-cli: + '0.1': + deprecated: false + module: + applications: + - app_type: apptainer + container: + path: docker://ghcr.io/epics-containers/edge-containers-cli + version: 4.4.1 + entrypoints: + - command: ec + name: ec + options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + - command: bash + name: ec-bash + options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: + - kubectl + - helm + - kubelogin + - klogout + - kustomize + - kubeseal + - kubectl-oidc_login + mounts: + - /dls/science/users/:/dls/science/users/ + - /scratch:/scratch + - /dls_sw/apps:/dls_sw/apps + - app_type: shell + name: ec-login + script: + - argocd login argocd.diamond.ac.uk --grpc-web --sso + dependencies: [] + description: ec command line tool for kubernetes IOCs + env_vars: + - name: EC_CLI_BACKEND + value: K8S + - name: EC_LOG_URL + value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* + - name: EC_TARGET + value: p47-beamline + - name: EC_SERVICES_REPO + value: https://github.com/epics-containers/p47-services + name: edge-containers-cli + version: '0.1' + example-module-apps: + '0.1': + deprecated: false + module: + applications: + - app_type: apptainer + container: + path: docker://ghcr.io/apptainer/lolcow + version: latest + entrypoints: + - command: cowsay + name: cowsay-hello + options: + apptainer_args: '' + command_args: Hello + host_binaries: [] + mounts: [] + - command: ls + name: show-directory + options: + apptainer_args: '' + command_args: -al /dls_sw_test + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: + - /dls_sw:/dls_sw_test:ro + - app_type: shell + name: test-echo-module-folder + script: + - echo $OTHER_VALUE + - app_type: shell + name: test-shell-script + script: + - echo This is the first line of a shell script + - echo and this is the second line. + - echo Your input is ${1} + dependencies: [] + description: Demonstration of a module configuration folder + env_vars: + - name: OTHER_VALUE + value: Test message OTHER_VALUE from example-module-folder + name: example-module-apps + version: '0.1' + example-module-deps: + '0.2': + deprecated: false + module: + applications: [] + dependencies: + - name: dls-pmac-control + version: '0.1' + - name: example-module-apps + version: '0.1' + description: Demonstration of deploy-tools dependencies + env_vars: [] + name: example-module-deps + version: '0.2' + phoebus: + '0.1': + deprecated: false + module: + applications: + - app_type: apptainer + container: + path: docker://ghcr.io/epics-containers/ec-phoebus + version: 4.7.3ec2 + entrypoints: + - command: java -jar /phoebus/phoebus.jar -server 7010 + name: phoebus + options: + apptainer_args: --env DISPLAY=${DISPLAY} -e + command_args: -server 7010 + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + dependencies: [] + description: Containerised release of CSS Phoebus + env_vars: [] + name: phoebus + version: '0.1' +settings: + default_versions: + dls-pmac-control: '0.1' diff --git a/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version new file mode 100644 index 0000000..aac51e4 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.1 \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.1 b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.1 new file mode 120000 index 0000000..75b7024 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.1 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.2 b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.2 new file mode 120000 index 0000000..0c56729 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/0.2 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version new file mode 100644 index 0000000..aac51e4 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.1 \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/0.1 b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/0.1 new file mode 120000 index 0000000..00b90c0 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/0.1 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version new file mode 100644 index 0000000..aac51e4 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.1 \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-apps/0.1 b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/0.1 new file mode 120000 index 0000000..53b9057 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/0.1 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/example-module-apps/0.1/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version new file mode 100644 index 0000000..3d3b2e8 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.2 \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-deps/0.2 b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/0.2 new file mode 120000 index 0000000..cd04ecd --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/0.2 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/example-module-deps/0.2/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/phoebus/.version b/tests/samples/deploy-tools-output/modulefiles/phoebus/.version new file mode 100644 index 0000000..aac51e4 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/phoebus/.version @@ -0,0 +1,2 @@ +#%Module1.0 +set ModulesVersion 0.1 \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modulefiles/phoebus/0.1 b/tests/samples/deploy-tools-output/modulefiles/phoebus/0.1 new file mode 120000 index 0000000..5468b29 --- /dev/null +++ b/tests/samples/deploy-tools-output/modulefiles/phoebus/0.1 @@ -0,0 +1 @@ +/tmp/deploy-tools-output/modules/phoebus/0.1/modulefile \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control new file mode 100755 index 0000000..91f2762 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="" +# Additional arguments for apptainer +apptainer_args="-e" +# Sif file path +sif_file="$(dirname $0)/../sif_files/a2c305dc6e8732efbb76b32c89f884f4.sif" +# Command to run in container +command="dls-pmac-control" +# Options and arguments to pass to command +command_args="" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in ; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file new file mode 100755 index 0000000..c42c3b5 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file @@ -0,0 +1,3 @@ +#! /bin/bash + +echo $EXAMPLE_VALUE diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml new file mode 100644 index 0000000..6ba35a7 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/module.yaml @@ -0,0 +1,29 @@ +applications: +- app_type: apptainer + container: + path: docker://ghcr.io/diamondlightsource/dls-pmac-control + version: 3.2.0b1 + entrypoints: + - command: dls-pmac-control + name: dls-pmac-control + options: + apptainer_args: -e + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] +- app_type: shell + name: test-echo-module-file + script: + - echo $EXAMPLE_VALUE +dependencies: [] +description: Demonstration of the deploy-tools process +env_vars: +- name: EXAMPLE_VALUE + value: Test message EXAMPLE_VALUE from example-module-file version 0.1 +name: dls-pmac-control +version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile new file mode 100644 index 0000000..2105caa --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile @@ -0,0 +1,10 @@ +#%Module1.0 +## +## dls-pmac-control - Demonstration of the deploy-tools process +## +module-whatis "Demonstration of the deploy-tools process" + +setenv EXAMPLE_VALUE "Test message EXAMPLE_VALUE from example-module-file version 0.1" + + +prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control new file mode 100755 index 0000000..91f2762 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="" +# Additional arguments for apptainer +apptainer_args="-e" +# Sif file path +sif_file="$(dirname $0)/../sif_files/a2c305dc6e8732efbb76b32c89f884f4.sif" +# Command to run in container +command="dls-pmac-control" +# Options and arguments to pass to command +command_args="" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in ; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file new file mode 100755 index 0000000..c42c3b5 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file @@ -0,0 +1,3 @@ +#! /bin/bash + +echo $EXAMPLE_VALUE diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml new file mode 100644 index 0000000..3b0c95b --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/module.yaml @@ -0,0 +1,29 @@ +applications: +- app_type: apptainer + container: + path: docker://ghcr.io/diamondlightsource/dls-pmac-control + version: 3.2.0b1 + entrypoints: + - command: dls-pmac-control + name: dls-pmac-control + options: + apptainer_args: -e + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] +- app_type: shell + name: test-echo-module-file + script: + - echo $EXAMPLE_VALUE +dependencies: [] +description: Demonstration of the deploy-tools process +env_vars: +- name: EXAMPLE_VALUE + value: Test message EXAMPLE_VALUE from example-module-file version 0.2 +name: dls-pmac-control +version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile new file mode 100644 index 0000000..91d603f --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile @@ -0,0 +1,10 @@ +#%Module1.0 +## +## dls-pmac-control - Demonstration of the deploy-tools process +## +module-whatis "Demonstration of the deploy-tools process" + +setenv EXAMPLE_VALUE "Test message EXAMPLE_VALUE from example-module-file version 0.2" + + +prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec new file mode 100755 index 0000000..8dd3c33 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="/dls/science/users/:/dls/science/users/,/scratch:/scratch,/dls_sw/apps:/dls_sw/apps" +# Additional arguments for apptainer +apptainer_args="" +# Sif file path +sif_file="$(dirname $0)/../sif_files/4b34247055f8388a8877c90302ff30d2.sif" +# Command to run in container +command="ec" +# Options and arguments to pass to command +command_args="" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in kubectl helm kubelogin klogout kustomize kubeseal kubectl-oidc_login; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash new file mode 100755 index 0000000..d4264d8 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="/dls/science/users/:/dls/science/users/,/scratch:/scratch,/dls_sw/apps:/dls_sw/apps" +# Additional arguments for apptainer +apptainer_args="" +# Sif file path +sif_file="$(dirname $0)/../sif_files/4b34247055f8388a8877c90302ff30d2.sif" +# Command to run in container +command="bash" +# Options and arguments to pass to command +command_args="" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in kubectl helm kubelogin klogout kustomize kubeseal kubectl-oidc_login; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login new file mode 100755 index 0000000..735a1c7 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login @@ -0,0 +1,3 @@ +#! /bin/bash + +argocd login argocd.diamond.ac.uk --grpc-web --sso diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml new file mode 100644 index 0000000..6b563eb --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/module.yaml @@ -0,0 +1,52 @@ +applications: +- app_type: apptainer + container: + path: docker://ghcr.io/epics-containers/edge-containers-cli + version: 4.4.1 + entrypoints: + - command: ec + name: ec + options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + - command: bash + name: ec-bash + options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: + - kubectl + - helm + - kubelogin + - klogout + - kustomize + - kubeseal + - kubectl-oidc_login + mounts: + - /dls/science/users/:/dls/science/users/ + - /scratch:/scratch + - /dls_sw/apps:/dls_sw/apps +- app_type: shell + name: ec-login + script: + - argocd login argocd.diamond.ac.uk --grpc-web --sso +dependencies: [] +description: ec command line tool for kubernetes IOCs +env_vars: +- name: EC_CLI_BACKEND + value: K8S +- name: EC_LOG_URL + value: https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}* +- name: EC_TARGET + value: p47-beamline +- name: EC_SERVICES_REPO + value: https://github.com/epics-containers/p47-services +name: edge-containers-cli +version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile new file mode 100644 index 0000000..cdb6c54 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile @@ -0,0 +1,13 @@ +#%Module1.0 +## +## edge-containers-cli - ec command line tool for kubernetes IOCs +## +module-whatis "ec command line tool for kubernetes IOCs" + +setenv EC_CLI_BACKEND "K8S" +setenv EC_LOG_URL "https://graylog.diamond.ac.uk/search?rangetype=relative&fields=message%2Csource&width=1489&highlightMessage=&relative=172800&q=pod_name%3A{service_name}*" +setenv EC_TARGET "p47-beamline" +setenv EC_SERVICES_REPO "https://github.com/epics-containers/p47-services" + + +prepend-path PATH "/tmp/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello new file mode 100755 index 0000000..73e0667 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="/dls_sw:/dls_sw_test:ro" +# Additional arguments for apptainer +apptainer_args="" +# Sif file path +sif_file="$(dirname $0)/../sif_files/a553018b87553334a94c33cb04eb025c.sif" +# Command to run in container +command="cowsay" +# Options and arguments to pass to command +command_args="Hello" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in ; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory new file mode 100755 index 0000000..184da44 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="/dls_sw:/dls_sw_test:ro" +# Additional arguments for apptainer +apptainer_args="" +# Sif file path +sif_file="$(dirname $0)/../sif_files/a553018b87553334a94c33cb04eb025c.sif" +# Command to run in container +command="ls" +# Options and arguments to pass to command +command_args="-al /dls_sw_test" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in ; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder new file mode 100755 index 0000000..4a94969 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder @@ -0,0 +1,3 @@ +#! /bin/bash + +echo $OTHER_VALUE diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script new file mode 100755 index 0000000..eef8758 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script @@ -0,0 +1,5 @@ +#! /bin/bash + +echo This is the first line of a shell script +echo and this is the second line. +echo Your input is ${1} diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml new file mode 100644 index 0000000..496c098 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/module.yaml @@ -0,0 +1,43 @@ +applications: +- app_type: apptainer + container: + path: docker://ghcr.io/apptainer/lolcow + version: latest + entrypoints: + - command: cowsay + name: cowsay-hello + options: + apptainer_args: '' + command_args: Hello + host_binaries: [] + mounts: [] + - command: ls + name: show-directory + options: + apptainer_args: '' + command_args: -al /dls_sw_test + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: + - /dls_sw:/dls_sw_test:ro +- app_type: shell + name: test-echo-module-folder + script: + - echo $OTHER_VALUE +- app_type: shell + name: test-shell-script + script: + - echo This is the first line of a shell script + - echo and this is the second line. + - echo Your input is ${1} +dependencies: [] +description: Demonstration of a module configuration folder +env_vars: +- name: OTHER_VALUE + value: Test message OTHER_VALUE from example-module-folder +name: example-module-apps +version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile new file mode 100644 index 0000000..d75b2a4 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile @@ -0,0 +1,10 @@ +#%Module1.0 +## +## example-module-apps - Demonstration of a module configuration folder +## +module-whatis "Demonstration of a module configuration folder" + +setenv OTHER_VALUE "Test message OTHER_VALUE from example-module-folder" + + +prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-apps/0.1/entrypoints" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml new file mode 100644 index 0000000..7763904 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/module.yaml @@ -0,0 +1,10 @@ +applications: [] +dependencies: +- name: dls-pmac-control + version: '0.1' +- name: example-module-apps + version: '0.1' +description: Demonstration of deploy-tools dependencies +env_vars: [] +name: example-module-deps +version: '0.2' diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile new file mode 100644 index 0000000..50cb34e --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile @@ -0,0 +1,11 @@ +#%Module1.0 +## +## example-module-deps - Demonstration of deploy-tools dependencies +## +module-whatis "Demonstration of deploy-tools dependencies" + + +module load dls-pmac-control/0.1 +module load example-module-apps/0.1 + +prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-deps/0.2/entrypoints" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus b/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus new file mode 100755 index 0000000..8608be8 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus @@ -0,0 +1,44 @@ +#! /bin/bash + +# Arguments: +# ${@} = Remaining args to pass to the command + +# Halt on error +set -e + +# Mounts for container +mounts="" +# Additional arguments for apptainer +apptainer_args="--env DISPLAY=${DISPLAY} -e" +# Sif file path +sif_file="$(dirname $0)/../sif_files/433dd7c0f3803fb792bf46b323938691.sif" +# Command to run in container +command="java -jar /phoebus/phoebus.jar -server 7010" +# Options and arguments to pass to command +command_args="-server 7010" + +# Raise an error if sif file does not exist +if [[ ! -f ${sif_file} ]]; then + echo "ERROR: sif file ${sif_file} does not exist" 1>&2 + exit 1 +fi + +# add mounts of host binaries into /usr/bin +for i in ; do + binary=$(which $i) + mounts="${mounts},${binary}:/usr/bin/${i}" +done + +# Set up mounts if any have been configured +if [[ ! -z ${mounts} ]]; then + opts="-B ${mounts}" +fi + +opts=${opts}" --env DISPLAY=${DISPLAY}" +opts=${opts}" ${apptainer_args}" + + +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml new file mode 100644 index 0000000..c793ed1 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/module.yaml @@ -0,0 +1,23 @@ +applications: +- app_type: apptainer + container: + path: docker://ghcr.io/epics-containers/ec-phoebus + version: 4.7.3ec2 + entrypoints: + - command: java -jar /phoebus/phoebus.jar -server 7010 + name: phoebus + options: + apptainer_args: --env DISPLAY=${DISPLAY} -e + command_args: -server 7010 + host_binaries: [] + mounts: [] + global_options: + apptainer_args: '' + command_args: '' + host_binaries: [] + mounts: [] +dependencies: [] +description: Containerised release of CSS Phoebus +env_vars: [] +name: phoebus +version: '0.1' diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile b/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile new file mode 100644 index 0000000..e3ec729 --- /dev/null +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile @@ -0,0 +1,9 @@ +#%Module1.0 +## +## phoebus - Containerised release of CSS Phoebus +## +module-whatis "Containerised release of CSS Phoebus" + + + +prepend-path PATH "/tmp/deploy-tools-output/modules/phoebus/0.1/entrypoints" \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index 368f8da..81eebc2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -2,6 +2,7 @@ import sys import tempfile from pathlib import Path +from shutil import rmtree from conftest import run_cli from deploy_tools import __version__ @@ -19,10 +20,39 @@ def test_cli_version(): def test_schema(): with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) + # Generate up to date schema files run_cli("schema", tmp_path) + # Compare with the expected schema files for schema in tmp_path.glob("*.json"): expected = PATH_TO_SCHEMAS / schema.name if schema.read_text() != expected.read_text(): raise AssertionError(f"Schema file {expected} is out of date.") + + +def test_demo_configuration(samples: Path, demo_config: Path): + # use a fixed path for the demo configuration so that the outputs are consistent + out_folder = "deploy-tools-output" + temp_out = Path("/tmp") / out_folder + expected_out = samples / out_folder + + # make sure the output directory is empty and exists + rmtree(temp_out, ignore_errors=True) + temp_out.mkdir(exist_ok=True) + + # generate the demo configuration output + run_cli("sync", "--from-scratch", str(temp_out), str(demo_config)) + + # compare the output with the expected output + for expected in expected_out.glob("**/*"): + if expected.is_dir(): + continue + # check that the file exists in the output directory + out_file = temp_out / expected.relative_to(expected_out) + assert out_file.exists(), f"File {out_file} does not exist." + + # check that the file contents are the same + assert expected.read_text() == out_file.read_text(), ( + f"File {out_file} is different." + ) From 151319572a3fc7467ebee56db8ef1891a46d96eb Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 07:38:45 +0000 Subject: [PATCH 04/12] fix pre-commit issuesr --- src/deploy_tools/templater.py | 5 ++++- tests/generate_samples.sh | 1 - .../modulefiles/dls-pmac-control/.version | 2 +- .../modulefiles/edge-containers-cli/.version | 2 +- .../modulefiles/example-module-apps/.version | 2 +- .../modulefiles/example-module-deps/.version | 2 +- .../samples/deploy-tools-output/modulefiles/phoebus/.version | 2 +- .../dls-pmac-control/0.1/entrypoints/dls-pmac-control | 2 +- .../modules/dls-pmac-control/0.1/modulefile | 2 +- .../dls-pmac-control/0.2/entrypoints/dls-pmac-control | 2 +- .../modules/dls-pmac-control/0.2/modulefile | 2 +- .../modules/edge-containers-cli/0.1/entrypoints/ec | 2 +- .../modules/edge-containers-cli/0.1/entrypoints/ec-bash | 2 +- .../modules/edge-containers-cli/0.1/modulefile | 2 +- .../modules/example-module-apps/0.1/entrypoints/cowsay-hello | 2 +- .../example-module-apps/0.1/entrypoints/show-directory | 2 +- .../modules/example-module-apps/0.1/modulefile | 2 +- .../modules/example-module-deps/0.2/modulefile | 2 +- .../modules/phoebus/0.1/entrypoints/phoebus | 2 +- .../deploy-tools-output/modules/phoebus/0.1/modulefile | 2 +- 20 files changed, 22 insertions(+), 20 deletions(-) diff --git a/src/deploy_tools/templater.py b/src/deploy_tools/templater.py index 11d44f4..3c77daa 100644 --- a/src/deploy_tools/templater.py +++ b/src/deploy_tools/templater.py @@ -50,7 +50,10 @@ def create( open_mode = "w" if overwrite else "x" with open(output_file, open_mode) as f: - f.write(self._templates[template].render(**parameters)) + rendered = self._templates[template].render(**parameters) + # enforce single trailing newline for pre-commit goodness + rendered = rendered.strip() + "\n" + f.write(rendered) permissions = EXECUTABLE_PERMISSIONS if executable else DEFAULT_PERMISSIONS output_file.chmod(permissions) diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh index 5ab5832..4186a40 100755 --- a/tests/generate_samples.sh +++ b/tests/generate_samples.sh @@ -19,4 +19,3 @@ rm -rf ${TMP_DIR}/.git* rm -rf ${SAMPLES_DIR} mkdir -p ${SAMPLES_DIR} cp -r ${TMP_DIR} ${SAMPLES_DIR} - diff --git a/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version index aac51e4..d9b8f92 100644 --- a/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version +++ b/tests/samples/deploy-tools-output/modulefiles/dls-pmac-control/.version @@ -1,2 +1,2 @@ #%Module1.0 -set ModulesVersion 0.1 \ No newline at end of file +set ModulesVersion 0.1 diff --git a/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version index aac51e4..d9b8f92 100644 --- a/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version +++ b/tests/samples/deploy-tools-output/modulefiles/edge-containers-cli/.version @@ -1,2 +1,2 @@ #%Module1.0 -set ModulesVersion 0.1 \ No newline at end of file +set ModulesVersion 0.1 diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version index aac51e4..d9b8f92 100644 --- a/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-apps/.version @@ -1,2 +1,2 @@ #%Module1.0 -set ModulesVersion 0.1 \ No newline at end of file +set ModulesVersion 0.1 diff --git a/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version index 3d3b2e8..26a66b4 100644 --- a/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version +++ b/tests/samples/deploy-tools-output/modulefiles/example-module-deps/.version @@ -1,2 +1,2 @@ #%Module1.0 -set ModulesVersion 0.2 \ No newline at end of file +set ModulesVersion 0.2 diff --git a/tests/samples/deploy-tools-output/modulefiles/phoebus/.version b/tests/samples/deploy-tools-output/modulefiles/phoebus/.version index aac51e4..d9b8f92 100644 --- a/tests/samples/deploy-tools-output/modulefiles/phoebus/.version +++ b/tests/samples/deploy-tools-output/modulefiles/phoebus/.version @@ -1,2 +1,2 @@ #%Module1.0 -set ModulesVersion 0.1 \ No newline at end of file +set ModulesVersion 0.1 diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control index 91f2762..e09f6b4 100755 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/dls-pmac-control @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile index 2105caa..2f27e87 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/modulefile @@ -7,4 +7,4 @@ module-whatis "Demonstration of the deploy-tools process" setenv EXAMPLE_VALUE "Test message EXAMPLE_VALUE from example-module-file version 0.1" -prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints" diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control index 91f2762..e09f6b4 100755 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/dls-pmac-control @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile index 91d603f..f524b6a 100644 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/modulefile @@ -7,4 +7,4 @@ module-whatis "Demonstration of the deploy-tools process" setenv EXAMPLE_VALUE "Test message EXAMPLE_VALUE from example-module-file version 0.2" -prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints" diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec index 8dd3c33..02556e4 100755 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash index d4264d8..35c7727 100755 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-bash @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile index cdb6c54..a5fe676 100644 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/modulefile @@ -10,4 +10,4 @@ setenv EC_TARGET "p47-beamline" setenv EC_SERVICES_REPO "https://github.com/epics-containers/p47-services" -prepend-path PATH "/tmp/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints" diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello index 73e0667..8e6480e 100755 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/cowsay-hello @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory index 184da44..111f5a5 100755 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/show-directory @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile index d75b2a4..56623f3 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/modulefile @@ -7,4 +7,4 @@ module-whatis "Demonstration of a module configuration folder" setenv OTHER_VALUE "Test message OTHER_VALUE from example-module-folder" -prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-apps/0.1/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-apps/0.1/entrypoints" diff --git a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile index 50cb34e..0c553eb 100644 --- a/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile +++ b/tests/samples/deploy-tools-output/modules/example-module-deps/0.2/modulefile @@ -8,4 +8,4 @@ module-whatis "Demonstration of deploy-tools dependencies" module load dls-pmac-control/0.1 module load example-module-apps/0.1 -prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-deps/0.2/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/example-module-deps/0.2/entrypoints" diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus b/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus index 8608be8..0dc0fef 100755 --- a/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/entrypoints/phoebus @@ -41,4 +41,4 @@ opts=${opts}" ${apptainer_args}" if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then set -x fi -apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" \ No newline at end of file +apptainer exec ${opts} ${sif_file} ${command} ${command_args} "${@}" diff --git a/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile b/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile index e3ec729..29810fb 100644 --- a/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile +++ b/tests/samples/deploy-tools-output/modules/phoebus/0.1/modulefile @@ -6,4 +6,4 @@ module-whatis "Containerised release of CSS Phoebus" -prepend-path PATH "/tmp/deploy-tools-output/modules/phoebus/0.1/entrypoints" \ No newline at end of file +prepend-path PATH "/tmp/deploy-tools-output/modules/phoebus/0.1/entrypoints" From 6a87e177fac5acfd931aded413eec6bef712e4a7 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 07:53:13 +0000 Subject: [PATCH 05/12] add apptainer to github actions tests --- .github/workflows/_test.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index f652d41..b8e4ba6 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -51,7 +51,13 @@ jobs: pip-install: ".[dev]" - name: Run tests - run: tox -e tests + run: | + sudo apt update + sudo apt install -y software-properties-common + sudo add-apt-repository -y ppa:apptainer/ppa + sudo apt update + sudo apt install -y apptainer + tox -e tests - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 From ec1c3d23189a1c9974e9da0b8792ba92d0ca58c0 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 25 Apr 2025 05:33:46 +0000 Subject: [PATCH 06/12] add logging to shell_entrypoint with DEPLOY_TOOLS_VERBOSE --- src/deploy_tools/templates/shell_entrypoint | 4 ++++ .../dls-pmac-control/0.1/entrypoints/test-echo-module-file | 4 ++++ .../dls-pmac-control/0.2/entrypoints/test-echo-module-file | 4 ++++ .../modules/edge-containers-cli/0.1/entrypoints/ec-login | 4 ++++ .../0.1/entrypoints/test-echo-module-folder | 4 ++++ .../example-module-apps/0.1/entrypoints/test-shell-script | 4 ++++ 6 files changed, 24 insertions(+) diff --git a/src/deploy_tools/templates/shell_entrypoint b/src/deploy_tools/templates/shell_entrypoint index 5c83fa2..8bc7087 100644 --- a/src/deploy_tools/templates/shell_entrypoint +++ b/src/deploy_tools/templates/shell_entrypoint @@ -1,5 +1,9 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + {% for line in script -%} {{ line }} {% endfor %} diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file index c42c3b5..6da99f6 100755 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.1/entrypoints/test-echo-module-file @@ -1,3 +1,7 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + echo $EXAMPLE_VALUE diff --git a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file index c42c3b5..6da99f6 100755 --- a/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file +++ b/tests/samples/deploy-tools-output/modules/dls-pmac-control/0.2/entrypoints/test-echo-module-file @@ -1,3 +1,7 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + echo $EXAMPLE_VALUE diff --git a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login index 735a1c7..b150b79 100755 --- a/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login +++ b/tests/samples/deploy-tools-output/modules/edge-containers-cli/0.1/entrypoints/ec-login @@ -1,3 +1,7 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + argocd login argocd.diamond.ac.uk --grpc-web --sso diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder index 4a94969..0a367dd 100755 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-echo-module-folder @@ -1,3 +1,7 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + echo $OTHER_VALUE diff --git a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script index eef8758..6ca6871 100755 --- a/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script +++ b/tests/samples/deploy-tools-output/modules/example-module-apps/0.1/entrypoints/test-shell-script @@ -1,5 +1,9 @@ #! /bin/bash +if [[ -n "${DEPLOY_TOOLS_VERBOSE}" ]]; then + set -x +fi + echo This is the first line of a shell script echo and this is the second line. echo Your input is ${1} From 72c1e586ebf5ec25c7514d4cca99bd61a6b671df Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 25 Apr 2025 05:40:17 +0000 Subject: [PATCH 07/12] clean up after test_demo_configuration --- tests/test_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_cli.py b/tests/test_cli.py index 81eebc2..37a18c2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -56,3 +56,5 @@ def test_demo_configuration(samples: Path, demo_config: Path): assert expected.read_text() == out_file.read_text(), ( f"File {out_file} is different." ) + + rmtree(temp_out, ignore_errors=True) From 68e026c43cee4ae2dbd5ff2de61bcd8bc9d80bf1 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 25 Apr 2025 05:44:24 +0000 Subject: [PATCH 08/12] add a task to run generate_samples.sh --- .vscode/tasks.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index da89333..f60d802 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -42,6 +42,15 @@ "command": "deploy-tools compare ${input:compare-mode} ${input:deploy-folder}", "problemMatcher": [] }, + { + "type": "shell", + "label": "Recreate tests sample output from demo_configuration", + "command": "tests/generate_samples.sh", + "options": { + "cwd": "${workspaceRoot}" + }, + "problemMatcher": [], + }, ], "inputs": [ { From db38ad193802b073140d134852a1b0a79f52e1b3 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 15:12:58 +0000 Subject: [PATCH 09/12] move schema path into conftest.py --- tests/conftest.py | 5 +++++ tests/test_cli.py | 8 ++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e44d8ce..2db9314 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,3 +43,8 @@ def samples(): @pytest.fixture def demo_config(): return Path(__file__).parent.parent / "src" / "deploy_tools" / "demo_configuration" + + +@pytest.fixture +def schemas(): + return Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" diff --git a/tests/test_cli.py b/tests/test_cli.py index 37a18c2..31e378a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,17 +7,13 @@ from conftest import run_cli from deploy_tools import __version__ -PATH_TO_SCHEMAS = ( - Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" -) - def test_cli_version(): cmd = [sys.executable, "-m", "deploy_tools", "--version"] assert subprocess.check_output(cmd).decode().strip() == __version__ -def test_schema(): +def test_schema(schemas: Path): with tempfile.TemporaryDirectory() as tmp: tmp_path = Path(tmp) @@ -26,7 +22,7 @@ def test_schema(): # Compare with the expected schema files for schema in tmp_path.glob("*.json"): - expected = PATH_TO_SCHEMAS / schema.name + expected = schemas / schema.name if schema.read_text() != expected.read_text(): raise AssertionError(f"Schema file {expected} is out of date.") From fedd0f6294e0e56a9af4e9f190c82c9cc8f7493f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Thu, 24 Apr 2025 07:27:34 +0000 Subject: [PATCH 10/12] add test for demo_configuration --- tests/conftest.py | 9 +++++++++ tests/generate_samples.sh | 4 ++++ tests/test_cli.py | 3 +++ 3 files changed, 16 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 2db9314..3c0789e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -48,3 +48,12 @@ def demo_config(): @pytest.fixture def schemas(): return Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" + + +def samples(): + return Path(__file__).parent / "samples" + + +@pytest.fixture +def demo_config(): + return Path(__file__).parent.parent / "src" / "deploy_tools" / "demo_configuration" diff --git a/tests/generate_samples.sh b/tests/generate_samples.sh index 4186a40..736c713 100755 --- a/tests/generate_samples.sh +++ b/tests/generate_samples.sh @@ -19,3 +19,7 @@ rm -rf ${TMP_DIR}/.git* rm -rf ${SAMPLES_DIR} mkdir -p ${SAMPLES_DIR} cp -r ${TMP_DIR} ${SAMPLES_DIR} +<<<<<<< HEAD +======= + +>>>>>>> ce3367b (add test for demo_configuration) diff --git a/tests/test_cli.py b/tests/test_cli.py index 31e378a..7bc103b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -52,5 +52,8 @@ def test_demo_configuration(samples: Path, demo_config: Path): assert expected.read_text() == out_file.read_text(), ( f"File {out_file} is different." ) +<<<<<<< HEAD rmtree(temp_out, ignore_errors=True) +======= +>>>>>>> ce3367b (add test for demo_configuration) From de548d2c5430a554c4d1e5a53c527b0e3c72be29 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 25 Apr 2025 13:48:12 +0000 Subject: [PATCH 11/12] fix merge error --- tests/conftest.py | 9 --------- tests/test_cli.py | 3 --- 2 files changed, 12 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3c0789e..9a58980 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,20 +40,11 @@ def samples(): return Path(__file__).parent / "samples" -@pytest.fixture -def demo_config(): - return Path(__file__).parent.parent / "src" / "deploy_tools" / "demo_configuration" - - @pytest.fixture def schemas(): return Path(__file__).parent.parent / "src" / "deploy_tools" / "models" / "schemas" -def samples(): - return Path(__file__).parent / "samples" - - @pytest.fixture def demo_config(): return Path(__file__).parent.parent / "src" / "deploy_tools" / "demo_configuration" diff --git a/tests/test_cli.py b/tests/test_cli.py index 7bc103b..31e378a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -52,8 +52,5 @@ def test_demo_configuration(samples: Path, demo_config: Path): assert expected.read_text() == out_file.read_text(), ( f"File {out_file} is different." ) -<<<<<<< HEAD rmtree(temp_out, ignore_errors=True) -======= ->>>>>>> ce3367b (add test for demo_configuration) From 1a881c32a4f401a7e5da198d533bdf7943e8a40f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Fri, 25 Apr 2025 15:11:51 +0000 Subject: [PATCH 12/12] use Annotated[] instead of Field() --- src/deploy_tools/models/apptainer_app.py | 46 +++++++++++++----------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/src/deploy_tools/models/apptainer_app.py b/src/deploy_tools/models/apptainer_app.py index 541fd16..dda8411 100644 --- a/src/deploy_tools/models/apptainer_app.py +++ b/src/deploy_tools/models/apptainer_app.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import Literal +from typing import Annotated, Literal from pydantic import Field @@ -7,25 +7,31 @@ class EntrypointOptions(ParentModel): - apptainer_args: str = Field( - description="Apptainer arguments to pass when launching the container", - default="", - ) - command_args: str = Field( - description="Arguments to pass to command entrypoint", - default="", - ) - mounts: Sequence[str] = Field( - description="A list of mount points to add to the container in the form of " - "'host_path:container_path'", - default=[], - ) - host_binaries: Sequence[str] = Field( - description="A list of host binaries to mount into the container. These are " - "discovered on the host using the current PATH and are mounted into the " - "container at /usr/bin/.", - default=[], - ) + apptainer_args: Annotated[ + str, + Field(description="Apptainer arguments to pass when launching the container"), + ] = "" + + command_args: Annotated[ + str, Field(description="Arguments to pass to command entrypoint") + ] = "" + + mounts: Annotated[ + Sequence[str], + Field( + description="A list of mount points to add to the container in the form of " + "'host_path:container_path'" + ), + ] = [] + + host_binaries: Annotated[ + Sequence[str], + Field( + description="A list of host binaries to mount into the container. " + "These are discovered on the host using the current PATH and are " + "mounted into the container at /usr/bin/." + ), + ] = [] class Entrypoint(ParentModel):