Shell script wrappers (#586)

* proof of concept for shell wrapper!

I still need to do the container command, exec, run, and the two inspects,
and then the same for docker. This is a WIP - the proof of concept seems to
be okay so far!

* adding reminder of singularity wrappers
* adding docker wrapper scripts and tests
* singularity exec/run should not have -s shell
* ensure template is fully loaded
* fixing bug that filesystem wrapper returns the filename instead of the content!
* update test: load_wrapper_script should have equivalent behavior regardless!
* Update shpc/main/modules/templates/singularity.tcl

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

shpc/client/sync.py

@@ -2,9 +2,10 @@
 __copyright__ = "Copyright 2021-2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
+import os
+
 import shpc.logger as logger
 import shpc.utils
-import os
 
 
 def sync_registry(args, parser, extra, subparser):

shpc/main/modules/templates/docker.lua

@@ -53,17 +53,13 @@ local containerPath = '{{ image }}'
 
 -- service environment variable to access docker URI
 setenv("PODMAN_CONTAINER", containerPath)
-set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
 
 local shellCmd = "{{ command }} ${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} ${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm --entrypoint {{ shell }} {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
 -- execCmd needs entrypoint to be the executor
 local execCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} "
 local runCmd = "{{ command }} ${PODMAN_OPTS} run ${PODMAN_COMMAND_OPTS} -i{% if settings.enable_tty %}t{% endif %} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file " .. moduleDir .. "/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v ${PWD} -w ${PWD} " .. containerPath
 local inspectCmd = "{{ command }} ${PODMAN_OPTS} inspect ${PODMAN_COMMAND_OPTS} " .. containerPath 
 
--- set_shell_function takes bashStr and cshStr
-set_shell_function("{|module_name|}-shell", shellCmd,  shellCmd)
-
 -- conflict with modules with the same name
 conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})
 
@@ -80,14 +76,19 @@ if (myShellName() == "bash") then
 {% endfor %}
 end{% endif %}
 
+{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
+
+-- set_shell_function takes bashStr and cshStr
+set_shell_function("{|module_name|}-shell", shellCmd,  shellCmd)
+
 -- A customizable exec function
 set_shell_function("{|module_name|}-exec", execCmd .. " --entrypoint \"\" " .. containerPath .. " \"$@\"",  execCmd .. " --entrypoint \"\" " .. containerPath)
 
 -- Always provide a container run
 set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"",  runCmd)
 
 -- Inspect runscript or deffile easily!
-set_shell_function("{|module_name|}-inspect", inspectCmd,  inspectCmd)
+set_shell_function("{|module_name|}-inspect", inspectCmd,  inspectCmd){% endif %}
 
 whatis("Name        : " .. myModuleName())
 whatis("Version     : " .. myModuleVersion())

shpc/main/modules/templates/docker.tcl

@@ -70,7 +70,6 @@ conflict {{ parsed_name.tool }}
 
 # service environment variable to access full SIF image path
 setenv PODMAN_CONTAINER "${containerPath}"
-set-alias {|module_name|}-container "echo ${containerPath}"
 
 # interactive shell to any container, plus exec for aliases
 set shellCmd "{{ command }} \${PODMAN_OPTS} run \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm -i{% if settings.enable_tty %}t{% endif %} --entrypoint {{ shell }} {% if settings.environment_file %}--env-file ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}" 
@@ -80,9 +79,6 @@ set execCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{%
 set runCmd "{{ command }} \${PODMAN_OPTS} run -i{% if settings.enable_tty %}t{% endif %} \${PODMAN_COMMAND_OPTS} -u `id -u`:`id -g` --rm {% if settings.environment_file %}--env-file  ${moduleDir}/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-v {{ settings.bindpaths }} {% endif %}{% if features.home %}-v {{ features.home }} {% endif %} -v $workdir -w $workdir ${containerPath}"
 set inspectCmd "{{ command }} \${PODMAN_OPTS} inspect ${containerPath}" 
 
-# set_shell_function takes bashStr and cshStr
-set-alias {|module_name|}-shell "${shellCmd}"
-
 # wrapper scripts? Add bin to path
 {% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}
 
@@ -101,6 +97,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
 {% endfor %}
 }{% endif %}
 
+{% if wrapper_scripts %}{% else %}
+set-alias {|module_name|}-container "echo ${containerPath}"
+
+set-alias {|module_name|}-shell "${shellCmd}"
+
 # A customizable exec function
 if { [ module-info shell bash ] } {
   set-alias {|module_name|}-exec "${execCmd} --entrypoint \"\" ${containerPath} \"\$@\""
@@ -116,7 +117,7 @@ if { [ module-info shell bash ] } {
 }
 
 # Inspect runscript or deffile easily!
-set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"
+set-alias {|module_name|}-inspect "${inspectCmd} ${containerPath}"{% endif %}
 
 #=====
 # Module options

shpc/main/modules/templates/singularity.lua

@@ -56,17 +56,13 @@ if not os.getenv("SINGULARITY_COMMAND_OPTS") then setenv ("SINGULARITY_COMMAND_O
 local containerPath = '{{ container_sif }}'
 -- service environment variable to access full SIF image path
 setenv("SINGULARITY_CONTAINER", containerPath)
-set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
 
 -- interactive shell to any container, plus exec for aliases
 local shellCmd = "singularity ${SINGULARITY_OPTS} shell ${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
 local execCmd = "singularity ${SINGULARITY_OPTS} exec ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
 local runCmd = "singularity ${SINGULARITY_OPTS} run ${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home }} --home {{ features.home }} {% endif %}{% if features.x11 %}-B {{ features.x11 }} {% endif %}{% if settings.environment_file %}-B " .. moduleDir .. "/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} " .. containerPath
 local inspectCmd = "singularity ${SINGULARITY_OPTS} inspect ${SINGULARITY_COMMAND_OPTS} " 
 
--- set_shell_function takes bashStr and cshStr
-set_shell_function("{|module_name|}-shell", shellCmd,  shellCmd)
-
 -- conflict with modules with the same name
 conflict("{{ parsed_name.tool }}"{% if name != parsed_name.tool %},"{{ name }}"{% endif %}{% if aliases %}{% for alias in aliases %}{% if alias.name != parsed_name.tool %},"{{ alias.name }}"{% endif %}{% endfor %}{% endif %})
 
@@ -83,6 +79,12 @@ if (myShellName() == "bash") then
 {% endfor %}
 end{% endif %}
 
+-- Only set shell functions if we don't use wrapper scripts
+{% if wrapper_scripts %}{% else %}set_shell_function("{|module_name|}-container", "echo " .. containerPath, "echo " .. containerPath)
+
+-- set_shell_function takes bashStr and cshStr
+set_shell_function("{|module_name|}-shell", shellCmd,  shellCmd)
+
 -- A customizable exec function
 set_shell_function("{|module_name|}-exec", execCmd .. containerPath .. " \"$@\"",  execCmd .. containerPath)
 
@@ -91,7 +93,8 @@ set_shell_function("{|module_name|}-run", runCmd .. " \"$@\"",  runCmd)
 
 -- Inspect runscript or deffile easily!
 set_shell_function("{|module_name|}-inspect-runscript", inspectCmd .. " -r  " .. containerPath,  inspectCmd .. containerPath)
-set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d  " .. containerPath,  inspectCmd .. containerPath)
+set_shell_function("{|module_name|}-inspect-deffile", inspectCmd .. " -d  " .. containerPath,  inspectCmd .. containerPath){% endif %}
+
 
 whatis("Name        : " .. myModuleName())
 whatis("Version     : " .. myModuleVersion())

shpc/main/modules/templates/singularity.tcl

@@ -78,18 +78,13 @@ setenv SINGULARITY_SHELL {{ settings.singularity_shell }}
 
 # service environment variable to access full SIF image path
 setenv SINGULARITY_CONTAINER "${containerPath}"
-set-alias {|module_name|}-container "echo ${containerPath}"
 
 # interactive shell to any container, plus exec for aliases
 set shellCmd "singularity \${SINGULARITY_OPTS} shell \${SINGULARITY_COMMAND_OPTS} -s {{ settings.singularity_shell }} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
 set execCmd "singularity \${SINGULARITY_OPTS} exec \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} "
 set runCmd "singularity \${SINGULARITY_OPTS} run \${SINGULARITY_COMMAND_OPTS} {% if features.gpu %}{{ features.gpu }} {% endif %}{% if features.home %}-B {{ features.home | replace("$", "\$") }} --home {{ features.home | replace("$", "\$") }} {% endif %}{% if features.x11 %}-B {{ features.x11 | replace("$", "\$") }} {% endif %}{% if settings.environment_file %}-B ${moduleDir}/{{ settings.environment_file }}:/.singularity.d/env/{{ settings.environment_file }}{% endif %} {% if settings.bindpaths %}-B {{ settings.bindpaths }}{% endif %} ${containerPath}"
 set inspectCmd "singularity \${SINGULARITY_OPTS} inspect \${SINGULARITY_COMMAND_OPTS} " 
 
-# set_shell_function takes bashStr and cshStr
-set-alias {|module_name|}-shell "${shellCmd}"
-
-
 # if we have any wrapper scripts, add bin to path
 {% if wrapper_scripts %}prepend-path PATH "${moduleDir}/bin"{% endif %}
 
@@ -108,7 +103,11 @@ set-alias {|module_name|}-shell "${shellCmd}"
 {% endfor %}
 }{% endif %}
 
-# A customizable exec function
+{% if wrapper_scripts %}{% else %}
+set-alias {|module_name|}-shell "${shellCmd}"
+set-alias {|module_name|}-container "echo ${containerPath}"
+
+
 if { [ module-info shell bash ] } {
   set-alias {|module_name|}-exec "${execCmd} ${containerPath} \"\$@\""
 } else {
@@ -124,8 +123,7 @@ if { [ module-info shell bash ] } {
 
 # Inspect runscript or deffile easily!
 set-alias {|module_name|}-inspect-runscript "${inspectCmd} -r ${containerPath}"
-set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"
-
+set-alias {|module_name|}-inspect-deffile "${inspectCmd} -d ${containerPath}"{% endif %}
 
 #=====
 # Module options

shpc/main/registry/filesystem.py

@@ -58,7 +58,7 @@ def load_wrapper_script(self, container_tech, script):
         """
         wrapper_script = self.find_wrapper_script(container_tech, script)
         if wrapper_script:
-            return os.path.join(self.dirname, wrapper_script)
+            return shpc.utils.read_file(os.path.join(self.dirname, wrapper_script))
 
     def override_exists(self, tag):
         """

shpc/main/wrappers/__init__.py

@@ -2,21 +2,18 @@
 __copyright__ = "Copyright 2022, Vanessa Sochat"
 __license__ = "MPL 2.0"
 
-import os
-
-from shpc.logger import logger
 
+from . import generators as gen
 from .base import WrapperScript
 
-here = os.path.abspath(os.path.dirname(__file__))
-
 
 def generate(image, container, config, **kwargs):
     """
     Generate one or more wrapper scripts for a container.
 
     Required arguments for all include container (class), image (path), settings
-    All kwargs go in optional. And this can be extended to include custom arguments.
+    All kwargs go in optional. The core set of constructor kwargs are provided
+    to each wrapper generator. This can be extended to include custom arguments.
     """
     # Return list of generated templates
     generated = []
@@ -33,46 +30,30 @@ def generate(image, container, config, **kwargs):
     }
 
     # Default wrapper for container technology, used for aliases unless overridden
+    default_wrapper = load_default_wrapper(constructor_kwargs)
+
+    # Generate wrappers for command aliases
+    generated += gen.alias_wrappers(aliases, default_wrapper, constructor_kwargs)
+
+    # Generate wrappers for container interactions
+    generated += gen.container_wrappers(constructor_kwargs)
+
+    # Container level wrapper scripts (allow eventually supporting custom podman)
+    generated += gen.custom_container_wrappers(constructor_kwargs)
+    return list(set(generated))
+
+
+def load_default_wrapper(constructor_kwargs):
+    """
+    Given a container and user settings, load a default wrapper.
+    """
     default_wrapper = None
+    container = constructor_kwargs["container"]
+    settings = constructor_kwargs["settings"]
+
     default_template_name = settings.wrapper_scripts.get(container.command)
     if default_template_name:
         default_wrapper = WrapperScript(default_template_name, **constructor_kwargs)
         # include_container_dir not set -> only look in the global locations
         default_wrapper.load_template()
-
-    # Command aliases
-    custom_wrapper_option_name = "%s_script" % container.templatefile
-    for alias in aliases:
-        # Allow overriding the template name in the script option
-        if custom_wrapper_option_name in alias:
-            wrapper = WrapperScript(
-                alias[custom_wrapper_option_name], **constructor_kwargs
-            )
-            wrapper.load_template(include_container_dir=True)
-        elif default_wrapper:
-            wrapper = default_wrapper
-        else:
-            logger.exit(
-                "Can't generate a wrapper script for '%s' as there is no template defined for %s"
-                % (alias["name"], container.templatefile)
-            )
-
-        # NB: alias is a dictionary
-        generated += wrapper.generate(alias["name"], alias)
-
-    # Container level wrapper scripts (allow eventually supporting custom podman)
-    scripts = {
-        "singularity": config.singularity_scripts,
-        "docker": config.docker_scripts,
-        "podman": config.docker_scripts,
-    }
-    # Additional commands defined in the custom container.yaml script section
-    listing = scripts.get(container.templatefile) or {}
-    for alias, template_name in listing.items():
-        wrapper = WrapperScript(template_name, **constructor_kwargs)
-        # Template wrapper scripts may live alongside container.yaml
-        wrapper.load_template(include_container_dir=True)
-        # NB: alias is a string
-        generated += wrapper.generate(alias, alias)
-
-    return list(set(generated))
+    return default_wrapper

shpc/main/wrappers/base.py

@@ -104,16 +104,21 @@ def load_template(self, include_container_dir=False):
         result = self.find_wrapper_script(template_paths, include_container_dir)
         loader = FileSystemLoader(template_paths)
         env = Environment(loader=loader)
+
+        # Do we have a filesystem path to load directly?
         if "path" in result:
             self.template = env.get_template(self.wrapper_template)
+
+        # Or string content to load?
         else:
             self.template = env.from_string(result["content"])
 
-    def generate(self, wrapper_name, alias_definition):
+    def generate(self, wrapper_name, alias_definition=None):
         """
         Template generation function.
         NB: alias_definition is a dictionary for command aliases, and a string
-        for additional arbitrary commands
+        for additional arbitrary commands. It is not required for container
+        interaction wrappers (e.g., exec, shell, etc.)
         """
 
         # Write scripts into container directory