Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Shell script wrappers #586

Merged
merged 10 commits into from
Sep 5, 2022
3 changes: 2 additions & 1 deletion shpc/client/sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
__copyright__ = "Copyright 2021-2022, Vanessa Sochat"
__license__ = "MPL 2.0"

import os

import shpc.logger as logger
import shpc.utils
import os


def sync_registry(args, parser, extra, subparser):
Expand Down
11 changes: 6 additions & 5 deletions shpc/main/modules/templates/docker.lua
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,13 @@ local containerPath = '{{ image }}'

-- service environment variable to access docker URI
setenv("PODMAN_CONTAINER", containerPath)
set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)

local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
-- execCmd needs entrypoint to be the executor
local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "
local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath

-- set_shell_function takes bashStr and cshStr
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)

-- conflict with modules with the same name
conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})

Expand All @@ -80,14 +76,19 @@ if (myShellName() == "bash") then
{% endfor %}
end{% endif %}

{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)

-- set_shell_function takes bashStr and cshStr
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)

-- A customizable exec function
set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " \"$@\"", execCmd .. " --entrypoint \"\" " .. containerPath)

-- Always provide a container run
set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd)

-- Inspect runscript or deffile easily!
set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd)
set_shell_function("{|module_name|}-inspect", inspectCmd, inspectCmd){% endif %}

whatis("Name : " .. myModuleName())
whatis("Version : " .. myModuleVersion())
Expand Down
11 changes: 6 additions & 5 deletions shpc/main/modules/templates/docker.tcl
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ conflict {{ parsed_name.tool }}

# service environment variable to access full SIF image path
setenv PODMAN_CONTAINER "${containerPath}"
set-alias {|module_name|}-container "echo ${containerPath}"

# interactive shell to any container, plus exec for aliases
set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
Expand All @@ -80,9 +79,6 @@ set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{%
set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}"

# set_shell_function takes bashStr and cshStr
set-alias {|module_name|}-shell "${shellCmd}"

# wrapper scripts? Add bin to path
{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}

Expand All @@ -101,6 +97,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
{% endfor %}
}{% endif %}

{% if wrapper_scripts %}{% else %}
set-alias {|module_name|}-container "echo ${containerPath}"

set-alias {|module_name|}-shell "${shellCmd}"

# A customizable exec function
if { [ module-info shell bash ] } {
set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath} \"\$@\""
Expand All @@ -116,7 +117,7 @@ if { [ module-info shell bash ] } {
}

# Inspect runscript or deffile easily!
set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"
set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"{% endif %}

#=====
# Module options
Expand Down
13 changes: 8 additions & 5 deletions shpc/main/modules/templates/singularity.lua
Original file line number Diff line number Diff line change
Expand Up @@ -56,17 +56,13 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O
local containerPath = '{{ container_sif }}'
-- service environment variable to access full SIF image path
setenv("SINGULARITY_CONTAINER", containerPath)
set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)

-- interactive shell to any container, plus exec for aliases
local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} "

-- set_shell_function takes bashStr and cshStr
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)

-- conflict with modules with the same name
conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})

Expand All @@ -83,6 +79,12 @@ if (myShellName() == "bash") then
{% endfor %}
end{% endif %}

-- Only set shell functions if we don't use wrapper scripts
{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)

-- set_shell_function takes bashStr and cshStr
set_shell_function("{|module_name|}-shell", shellCmd, shellCmd)

-- A customizable exec function
set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " \"$@\"", execCmd .. containerPath)

Expand All @@ -91,7 +93,8 @@ set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"", runCmd)

-- Inspect runscript or deffile easily!
set_shell_function("{|module_name|}-inspect-runscript", inspectCmd .. " -r " .. containerPath, inspectCmd .. containerPath)
set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath)
set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d " .. containerPath, inspectCmd .. containerPath){% endif %}


whatis("Name : " .. myModuleName())
whatis("Version : " .. myModuleVersion())
Expand Down
14 changes: 6 additions & 8 deletions shpc/main/modules/templates/singularity.tcl
Original file line number Diff line number Diff line change
Expand Up @@ -78,18 +78,13 @@ setenv SINGULARITY_SHELL {{ settings.singularity_shell }}

# service environment variable to access full SIF image path
setenv SINGULARITY_CONTAINER "${containerPath}"
set-alias {|module_name|}-container "echo ${containerPath}"

# interactive shell to any container, plus exec for aliases
set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} "

# set_shell_function takes bashStr and cshStr
set-alias {|module_name|}-shell "${shellCmd}"


# if we have any wrapper scripts, add bin to path
{% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}

Expand All @@ -108,7 +103,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
{% endfor %}
}{% endif %}

# A customizable exec function
{% if wrapper_scripts %}{% else %}
set-alias {|module_name|}-shell "${shellCmd}"
set-alias {|module_name|}-container "echo ${containerPath}"


if { [ module-info shell bash ] } {
set-alias {|module_name|}-exec "${execCmd} ${containerPath} \"\$@\""
} else {
Expand All @@ -124,8 +123,7 @@ if { [ module-info shell bash ] } {

# Inspect runscript or deffile easily!
set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}"
set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"

set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"{% endif %}

#=====
# Module options
Expand Down
2 changes: 1 addition & 1 deletion shpc/main/registry/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def load_wrapper_script(self, container_tech, script):
"""
wrapper_script = self.find_wrapper_script(container_tech, script)
if wrapper_script:
return os.path.join(self.dirname, wrapper_script)
return shpc.utils.read_file(os.path.join(self.dirname, wrapper_script))

def override_exists(self, tag):
"""
Expand Down
67 changes: 24 additions & 43 deletions shpc/main/wrappers/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,21 +2,18 @@
__copyright__ = "Copyright 2022, Vanessa Sochat"
__license__ = "MPL 2.0"

import os

from shpc.logger import logger

from . import generators as gen
from .base import WrapperScript

here = os.path.abspath(os.path.dirname(__file__))


def generate(image, container, config, **kwargs):
"""
Generate one or more wrapper scripts for a container.
Required arguments for all include container (class), image (path), settings
All kwargs go in optional. And this can be extended to include custom arguments.
All kwargs go in optional. The core set of constructor kwargs are provided
to each wrapper generator. This can be extended to include custom arguments.
"""
# Return list of generated templates
generated = []
Expand All @@ -33,46 +30,30 @@ def generate(image, container, config, **kwargs):
}

# Default wrapper for container technology, used for aliases unless overridden
default_wrapper = load_default_wrapper(constructor_kwargs)

# Generate wrappers for command aliases
generated += gen.alias_wrappers(aliases, default_wrapper, constructor_kwargs)

# Generate wrappers for container interactions
generated += gen.container_wrappers(constructor_kwargs)

# Container level wrapper scripts (allow eventually supporting custom podman)
generated += gen.custom_container_wrappers(constructor_kwargs)
return list(set(generated))


def load_default_wrapper(constructor_kwargs):
"""
Given a container and user settings, load a default wrapper.
"""
default_wrapper = None
container = constructor_kwargs["container"]
settings = constructor_kwargs["settings"]

default_template_name = settings.wrapper_scripts.get(container.command)
if default_template_name:
default_wrapper = WrapperScript(default_template_name, **constructor_kwargs)
# include_container_dir not set -> only look in the global locations
default_wrapper.load_template()

# Command aliases
custom_wrapper_option_name = "%s_script" % container.templatefile
for alias in aliases:
# Allow overriding the template name in the script option
if custom_wrapper_option_name in alias:
wrapper = WrapperScript(
alias[custom_wrapper_option_name], **constructor_kwargs
)
wrapper.load_template(include_container_dir=True)
elif default_wrapper:
wrapper = default_wrapper
else:
logger.exit(
"Can't generate a wrapper script for '%s' as there is no template defined for %s"
% (alias["name"], container.templatefile)
)

# NB: alias is a dictionary
generated += wrapper.generate(alias["name"], alias)

# Container level wrapper scripts (allow eventually supporting custom podman)
scripts = {
"singularity": config.singularity_scripts,
"docker": config.docker_scripts,
"podman": config.docker_scripts,
}
# Additional commands defined in the custom container.yaml script section
listing = scripts.get(container.templatefile) or {}
for alias, template_name in listing.items():
wrapper = WrapperScript(template_name, **constructor_kwargs)
# Template wrapper scripts may live alongside container.yaml
wrapper.load_template(include_container_dir=True)
# NB: alias is a string
generated += wrapper.generate(alias, alias)

return list(set(generated))
return default_wrapper
9 changes: 7 additions & 2 deletions shpc/main/wrappers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,16 +104,21 @@ def load_template(self, include_container_dir=False):
result = self.find_wrapper_script(template_paths, include_container_dir)
loader = FileSystemLoader(template_paths)
env = Environment(loader=loader)

# Do we have a filesystem path to load directly?
if "path" in result:
self.template = env.get_template(self.wrapper_template)

# Or string content to load?
else:
self.template = env.from_string(result["content"])

def generate(self, wrapper_name, alias_definition):
def generate(self, wrapper_name, alias_definition=None):
"""
Template generation function.
NB: alias_definition is a dictionary for command aliases, and a string
for additional arbitrary commands
for additional arbitrary commands. It is not required for container
interaction wrappers (e.g., exec, shell, etc.)
"""

# Write scripts into container directory
Expand Down
Loading