From 96526b45a679882c2a46f709ac50e0445245a88e Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Wed, 1 May 2024 13:38:06 -0500 Subject: [PATCH 1/9] copy over changes from before de-submodule transition --- CMakeLists.txt | 2 +- scripts/.shellcheckrc | 30 +++ scripts/README.md | 84 ++++++++ scripts/ldmx-denv-init.sh | 126 ++++++++++++ scripts/ldmx-denv-transition.sh | 326 ++++++++++++++++++++++++++++++++ 5 files changed, 567 insertions(+), 1 deletion(-) create mode 100644 scripts/.shellcheckrc create mode 100644 scripts/README.md create mode 100644 scripts/ldmx-denv-init.sh create mode 100644 scripts/ldmx-denv-transition.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6dcc269f9..f1c673c7c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ endif() # If a user is building outside of a Docker or Singularity environment, # warn them. -if (NOT EXISTS /.dockerenv AND NOT EXISTS /singularity) +if (NOT EXISTS /.dockerenv AND NOT EXISTS /singularity AND NOT EXISTS /run/.containerenv) message(WARNING "You are not inside a container; you may be working in an untested environment.") endif() diff --git a/scripts/.shellcheckrc b/scripts/.shellcheckrc new file mode 100644 index 000000000..ff8b14428 --- /dev/null +++ b/scripts/.shellcheckrc @@ -0,0 +1,30 @@ +# Overrides the shell detected from the shebang. +# This is useful for files meant to be included (and thus lacking a shebang), +# or possibly as a more targeted alternative to 'disable=SC2039'. +shell=sh +# You can override this configuration on the command line with --shell + +# Always allow ShellCheck to open arbitrary files from 'source' statements. +external-sources=true +check-sourced=true + +# Enable all optional checks +enable=all + +# We want the most picky shellcheck so we have it even flag style errors +severity=style + +# This function is invoked in an 'if' condition so set -e will be disabled. +# Invoke separately if failures should cause the script to exit. +# - We don't want to exit if errors happen inside a check, that's why we have a check... +disable=SC2310 + +# Bash implicitly disabled set -e for this function invocation because +# it's inside a command substitution. Add set -e; before it or enable inherit_errexit. +# - Don't care if we inherit errexit inside substitutions, we do checks for that. +disable=SC2311 + +# Consider invoking this command separately to avoid masking its return value +# (or use '|| true' to ignore). +# - We already check errors and adding "|| true" everywhere hinders readability. +disable=SC2312 diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..479d4312c --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,84 @@ +# LDMX Environment Initialization + +The LDMX development and running environment is defined as a container image. +Running this container image properly can be complicated on many systems, +so we've isolated the running of this container into shell scripts so that +the average user/developer does not need to worry about the intricacies +of container running. + +## Legacy Solution +The first development of this solution defined a series of functions in +`bash` that wrapped `docker` and `singularity` and focused on running the +image as built by [LDMX-Software/docker](https://github.com/LDMX-Software/docker). +This solution, while functional, has a few edge-case issues and is difficult +to test independently of ldmx-sw. +These issues motivated the development of a separate project which we can +now use for development of ldmx-sw. + +## denv +[denv](https://tomeichlersmith.github.io/denv/) +is a project to generalize the original bash functions to support +more shells, more systems, and more container runners. With a ground-up +redesign, it has benefited from lessons learned while using the original +bash functions and, as a independent project, can be thoroughly tested. + +Utilizing `denv` to develop and run ldmx-sw is a simpler task than +trying to use the container runners directly; however, it is a bit +more complicated than the original bash functions due to its generality. +This makes defining wrapping environment scripts a helpful tool for +easing the burden placed on any users. Below, the LDMX denv configuration +is documented to help users understand what is happening as well as +help anyone interested in developing a wrapping script to be used +within the shell of their choice. + +### Initialization +``` +denv \ + init \ # use denv special sub-command for initialization + --clean-env \ # avoid passing environment variables + --name "ldmx" \ # name this denv after our experiment :) + "ldmx/dev:latest" \ # use the dev image + # path to directory containing ldmx-sw +``` +For example, if you are in the ldmx-sw directory, +``` +denv init --clean-env --name ldmx ldmx/dev:latest .. +``` +The default path is the current directory, so you could also init immediately +after cloning ldmx-sw. +``` +git clone --recursive git@github.com:ldmx-software/ldmx-sw.git +denv init --clean-env --name ldmx ldmx/dev:latest +``` +The default `--name` of a denv is the directory where `init` was run, +so the final example (and the one I use on most of my machines) is +``` +mkdir ldmx +cd ldmx +git clone --recursive git@github.com:ldmx-software/ldmx-sw.git +denv init --clean-env ldmx/dev:latest +``` + +### Wrapping +Wrapping `denv` can have the benefit of reducing the amount of characters +needed for the average user to type when doing common tasks. Environment +scripts wrapping denv for a specific shell should do the following. + +- Deduce `LDMX_BASE` and define it as an environment variable for the user +- Make sure the LDMX denv is initialized for the user +- Define the `ldmx` function with CLI defined below + +### `ldmx` CLI Specification +v0.0.0: just an idea + +- `ldmx` + - `help` : print help message describing the CLI + - `config` : print shell version being used and run `denv config print` + - `use` : first argument is repo (`dev` or `pro`) and then second argument is tag (e.g. `latest`) + - deduce full docker image tag name from repo and tag and then if expand the tag to include + the path on `/cvmfs` (if available) or just the DockerHub tag (if not) + - `pull` : call `ldmx use` and then `denv config image pull` + - `mount` : pass arguments to `denv config mounts` + - `setenv` : pass arguments to `denv config env copy` + - `` : pass directly to `denv` + diff --git a/scripts/ldmx-denv-init.sh b/scripts/ldmx-denv-init.sh new file mode 100644 index 000000000..b6188b3ee --- /dev/null +++ b/scripts/ldmx-denv-init.sh @@ -0,0 +1,126 @@ +############################################################################### +# ldmx-denv-init.sh +# Make sure that the LDMX denv is initialized properly before proceeding. +# +# This script is desigend to be idempotent: i.e. if the denv is already +# properly configured, then this script should not have any effect. +############################################################################### + +# print each argument on its own line with the first line +# prefixed with "ERROR [ldmx-denv-init.sh]: ". +_error() { + printf >&2 "\033[1;31mERROR [ldmx-denv-init.sh]: \033[0m\033[31m%s\n" "$1" + shift + while [ "$#" -gt "0" ]; do + printf >&2 " %s\n" "$1" + shift + done + printf >&2 "\033[0m" +} + +# print the question passed to us and process user input +# continues in infinite loop if user is not explicitly answering yes or no +# Arguments +# 1 - question to ask the user (yes or no) +# Output +# returns 0 if user answers yes and 1 if user answers no +_user_confirm() { + question="$*" + while true; do + printf "\033[1m%s\033[0m [Y/n] " "${question}" + read -r ans + case "${ans}" in + Y|y|yes) + return 0 + ;; + N|n|no) + return 1 + ;; + *) + _error "${ans} is not one of Y, y, yes, N, n, or no." + ;; + esac + done +} + +if ! command -v denv > /dev/null 2>&1; then + _error "'denv' is not installed." + if _user_confirm "Do you wish to try to install 'denv' now?"; then + curl -s https://raw.githubusercontent.com/tomeichlersmith/denv/main/install | sh + else + printf "Not attempting to install 'denv'. Please follow the instructions at\n %s\n" \ + "https://tomeichlersmith.github.io/denv/getting_started.html#installation" + unset -f _error _user_confirm + return 1 + fi +fi + +if ! denv check --quiet; then + # maybe try loading apptainer via the 'module' command? + # https://github.com/LDMX-Software/ldmx-sw/issues/1248#issuecomment-1896618339 + denv check + _error "'denv' unable to find a supported container runner." \ + "Install one of the container runners 'denv' checked for above." + unset -f _error _user_confirm + return 2 +fi + +############################################################################### +# Attempt to deduce LDMX_BASE +# Unfortunately, deducing the path to a script being sourced is not possible +# using POSIX-compliant methods[1], so we need to fallback to separating +# different running scenarios. +# +# [1]: https://stackoverflow.com/a/29835459 +# +# Users can avoid all of this complication by pre-defining the LDMX_BASE +# environment variable. +############################################################################### +if [ -z "${LDMX_BASE+x}" ]; then + _full_path() ( + CDPATH= + cd -- "${1}" && pwd -P + ) + _base_from_script() { + _full_path "$(dirname -- "${1}")/../../" + } + if [ -n "${ZSH_VERSION}" ]; then + # zsh defines ZSH_EVAL_CONTEXT holding the details on how + # this code is being executed + # disable warnings about undefined variable and string indexing + # shellcheck disable=SC2154,SC3057 + LDMX_BASE="$(_base_from_script "${ZSH_EVAL_CONTEXT:file}")" + elif [ -n "${KSH_VERSION}" ]; then + # Korn defines the _ variable to the the file being sourced + # disable warning about undefined '_' + # shellcheck disable=SC3028 + LDMX_BASE="$(_base_from_script "${_}")" + elif [ -n "${BASH_VERSION}" ]; then + # running from bash, bash defines the BASH_SOURCE array to help us + # find the location of this file + # disable warning about undefined variable and array references + # shellcheck disable=SC3028,SC3054 + LDMX_BASE="$(_base_from_script "${BASH_SOURCE[0]}")" + else + # unable to deduce shell, resort to pwd + if expr "${PWD}" : ".*ldmx-sw$"; then + LDMX_BASE="$(_full_path ..)" + elif expr "${PWD}" : ".*ldmx-sw/scripts$"; then + LDMX_BASE="$(_full_path ../..)" + fi + fi + unset -f _full_path _base_from_script +fi +# re-export LDMX_BASE in case of user does inline variable definition like +# LDMX_BASE=/path/to/ldmx/base source /full/path/to/ldmx-denv-init.sh +export LDMX_BASE + +############################################################################### +# Check if the LDMX denv is initialized. If not, do a default initialization. +# TODO: update to `denv check --workspace` introduced in denv v0.7.0 to avoid +# reliance on internal implementation on how denv stores its config +############################################################################### +if [ ! -f "${LDMX_BASE}/.denv/config" ]; then + denv init --clean-env --name "ldmx" "ldmx/dev:latest" "${LDMX_BASE}" +fi +unset -f _error _user_confirm diff --git a/scripts/ldmx-denv-transition.sh b/scripts/ldmx-denv-transition.sh new file mode 100644 index 000000000..fadf58830 --- /dev/null +++ b/scripts/ldmx-denv-transition.sh @@ -0,0 +1,326 @@ +############################################################################### +# ldmx-denv-transition.sh +# Replicate some of the `ldmx` commands by simply wrapping the equivalent +# `denv` commands along with printouts explaining what is being done. +# so that users know that the same procedures can be done with `denv`. +############################################################################### + +# make sure denv is initialized +. "$(dirname "${BASH_SOURCE[0]}")/ldmx-denv-init.sh" + +# mimicing the ldmx bash function requires bash, point out user +# doesn't need bash anymore if they use denv directly +if [[ -z ${BASH} ]]; then + echo "[ldmx-env.sh] [ERROR] You aren't in a bash shell. You are in '$0'." + [[ "${SHELL}" = *"bash"* ]] || echo " Your default shell '${SHELL}' isn't bash." + cat <<\HELP + If you'd prefer to not use bash, you can use 'denv' directly rather than + this wrapper script. + From the ldmx-sw directory, you can setup a denv to use our development image. + + . scripts/ldmx-denv-init.sh + + And then run commands within it by prefixing them with 'denv': + + denv cmake -B build -S . + denv cmake --build build --target install + denv fire SimCore/test/basic.py + + Look at 'denv --help' for other denv-specific commands. +HELP + return 1 +fi + +# Use the input container and error out if not available +__ldmx_use() { + local _repo_name="$1" + local _image_tag="$2" + cmd="denv config image ldmx/${_repo_name}:${_image_tag}" + echo "${cmd}" + ${cmd} + return $? +} + +############################################################################### +# __ldmx_config +# Print the configuration of the current setup +############################################################################### +__ldmx_config() { + echo "LDMX base directory: ${LDMX_BASE}" + echo "uname: $(uname -a)" + echo "Bash version: ${BASH_VERSION}" + echo "denv config print" + denv config print + return $? +} + +############################################################################### +# __ldmx_mount +# Tell us to mount the passed directory to the container when we run +# By default, we already mount the LDMX_BASE directory, so none of +# its subdirectories need to (or should be) specified. +############################################################################### +__ldmx_mount() { + echo "denv config mounts $*" + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + denv config mounts $@ + return $? +} + +############################################################################### +# __ldmx_setenv +# Tell us to pass an environment variable to the container when we run +# By default, we pass the LDMX_BASE and DISPLAY variables explicitly, +# because their syntax is too different between docker and singularity, +# so none of these need to (or should be) specified. +############################################################################### +__ldmx_setenv() { + echo "denv config env copy $*" + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + denv config env copy $@ + return $? +} + +############################################################################### +# __ldmx_help +# Print some helpful message to the terminal +############################################################################### +__ldmx_help() { + cat <<\HELP + + USAGE: + ldmx [ ...] + + COMMANDS: + help : print this help message and exit + ldmx help + config : Print the current configuration of the container + ldmx config + use : Use the input repo and tag of the container for running + ldmx use (dev | pro | local) + pull : Pull down the input repo and tag of the container + ldmx pull (dev | pro | local) + mount : Attach the input directory to the container when running + ldmx mount + setenv : Set an environment variable in the container when running + ldmx setenv + : Run the input command in your current directory in the container + ldmx [ ...] + ldmx cmake .. + ldmx make install + ldmx fire config.py + ldmx python3 ana.py + +HELP + return 0 +} + +############################################################################### +# ldmx +# The root command for users interacting with the ldmx container environment. +# This function is really just focused on parsing CLI and going to the +# corresponding subcommand. +# +# There are lots of subcommands, go to those functions to learn the detail +# about them. +############################################################################### +ldmx() { + # if there are no arguments, print the help + [[ "$#" == "0" ]] && { __ldmx_help; return $?; } + # divide commands by number of arguments + cmd="__ldmx_${1}" + case "${1}" in + help|config) + if [[ "$#" != "1" ]]; then + ${cmd} + echo "ERROR: 'ldmx ${1}' takes no arguments. (It was ignored.)" + return 1 + fi + ${cmd} + return $? + ;; + list|mount|setenv|source) + if [[ "$#" != "2" ]]; then + __ldmx_help + echo "ERROR: ldmx ${1} takes one argument." + return 1 + fi + ${cmd} "$2" + return $? + ;; + pull) + if [[ "$#" != "3" ]]; then + __ldmx_help + echo "ERROR: 'ldmx pull' takes two arguments: ." + return 1 + fi + __ldmx_use "$2" "$3" + echo "denv config image pull" + denv config image pull + return $? + ;; + use) + if [[ "$#" != "3" ]]; then + __ldmx_help + echo "ERROR: 'ldmx use' takes two arguments: ." + return 1 + fi + __ldmx_use "$2" "$3" + return $? + ;; + *) + echo "denv $*" + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + denv $@ + return $? + ;; + esac +} + +############################################################################### +# DONE WITH NECESSARY PARTS +# Everything below here is icing on the usability cake. +############################################################################### + +############################################################################### +# Bash Tab Completion +# This next section is focused on setting up the infrastucture for smart +# tab completion with the ldmx command and its sub-commands. +############################################################################### + +############################################################################### +# __ldmx_complete_directory +# Some of our sub-commands take a directory as input. +# In these cases, we can pretend to cd and use bash's internal +# tab-complete functions. +# +# All this requires is for us to shift the COMP_WORDS array one to +# the left so that the bash internal tab-complete functions don't +# get distracted by our base command 'ldmx' at the front. +# +# We could allow for the shift to be more than one if there is a deeper +# tree of commands that need to be allowed in the futre. +############################################################################### +__ldmx_complete_directory() { + local _num_words="1" + #shellcheck disable=SC2206 + COMP_WORDS=(${COMP_WORDS[@]:_num_words}) + COMP_CWORD=$((COMP_CWORD - _num_words)) + _cd +} + +############################################################################### +# __ldmx_complete_command +# Tab-complete with a command used commonly inside the container +# +# Search the install location of ldmx-sw for ldmx-sw executables +# and include hard-coded common commands. Any strings passed are +# also included. +# +# Assumes current argument being tab completed is stored in +# bash variable 'curr_word'. +############################################################################### +__ldmx_complete_command() { + # generate up-to-date list of options + local _options="$* cmake make python3 root rootbrowse" + for ldmx_executable in "${LDMX_BASE}/ldmx-sw/install/bin"/*; do + _options="${_options} $(basename "${ldmx_executable}")" + done + + # match current word (perhaps empty) to the list of options + #shellcheck disable=SC2207 + COMPREPLY=($(compgen -W "${_options}" "${curr_word}")) +} + +############################################################################### +# __ldmx_complete_bash_default +# Restore the default tab-completion in bash that uses the readline function +# Bash default tab completion just looks for filenames +############################################################################### +__ldmx_complete_bash_default() { + compopt -o default + COMPREPLY=() +} + +############################################################################### +# __ldmx_dont_complete +# Don't tab complete or suggest anything if user s +############################################################################### +__ldmx_dont_complete() { + COMPREPLY=() +} + +############################################################################### +# Modify the list of completion options on the command line +# Helpful discussion of this procedure from a blog post +# https://iridakos.com/programming/2018/03/01/bash-programmable-completion-tutorial +# +# Helpful Stackoverflow answer +# https://stackoverflow.com/a/19062943 +# +# COMP_WORDS - bash array of space-separated command line inputs including base command +# COMP_CWORD - index of current word in argument list +# COMPREPLY - options available to user, if only one, auto completed +############################################################################### +__ldmx_complete() { + # disable readline filename completion + compopt +o default + + local curr_word="${COMP_WORDS[${COMP_CWORD}]}" + + if [[ "${COMP_CWORD}" = "1" ]]; then + # tab completing a main argument + __ldmx_complete_command "config pull use run mount setenv" + elif [[ "${COMP_CWORD}" = "2" ]]; then + # tab complete a sub-argument, + # depends on the main argument + case "${COMP_WORDS[1]}" in + config|setenv) + # no more arguments + __ldmx_dont_complete + ;; + pull|use) + # container repositories after these commands + #shellcheck disable=SC2207 + COMPREPLY=($(compgen -W "dev pro" "${_curr_word}")) + ;; + run|mount) + #directories only after these commands + __ldmx_complete_directory + ;; + *) + # files like normal tab complete after everything else + __ldmx_complete_bash_default + ;; + esac + else + # three or more arguments + # check base argument to see if we should continue + case "${COMP_WORDS[1]}" in + config|pull|use|mount|setenv) + # these commands shouldn't have tab complete for the third argument + # (or shouldn't have the third argument at all) + __ldmx_dont_complete + ;; + run) + if [[ "${COMP_CWORD}" = "3" ]]; then + # third argument to run should be an inside-container command + __ldmx_complete_command + else + # later arguments to run should be bash default + __ldmx_complete_bash_default + fi + ;; + *) + # everything else has bash default (filenames) + __ldmx_complete_bash_default + ;; + esac + fi +} + +# Tell bash the tab-complete options for our main function ldmx +complete -F __ldmx_complete ldmx From a6b21ac658101e3b657de2d54ee464111a0f2447 Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Mon, 6 May 2024 21:44:47 -0500 Subject: [PATCH 2/9] wip: wavering on which path to choose --- scripts/ldmx-denv-init.sh | 2 +- scripts/ldmx-denv-transition.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ldmx-denv-init.sh b/scripts/ldmx-denv-init.sh index b6188b3ee..afd921386 100644 --- a/scripts/ldmx-denv-init.sh +++ b/scripts/ldmx-denv-init.sh @@ -120,7 +120,7 @@ export LDMX_BASE # TODO: update to `denv check --workspace` introduced in denv v0.7.0 to avoid # reliance on internal implementation on how denv stores its config ############################################################################### -if [ ! -f "${LDMX_BASE}/.denv/config" ]; then +if ! denv check --workspace --quiet; then denv init --clean-env --name "ldmx" "ldmx/dev:latest" "${LDMX_BASE}" fi unset -f _error _user_confirm diff --git a/scripts/ldmx-denv-transition.sh b/scripts/ldmx-denv-transition.sh index fadf58830..727e4f396 100644 --- a/scripts/ldmx-denv-transition.sh +++ b/scripts/ldmx-denv-transition.sh @@ -18,7 +18,7 @@ if [[ -z ${BASH} ]]; then this wrapper script. From the ldmx-sw directory, you can setup a denv to use our development image. - . scripts/ldmx-denv-init.sh + denv init ldmx/dev:latest .. And then run commands within it by prefixing them with 'denv': From de4deb09256e25436734fcc0868077684be94867 Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 10:51:30 -0500 Subject: [PATCH 3/9] rename to reflect that we are wrapping denv for bash --- scripts/README.md | 4 +++- scripts/{ldmx-denv-transition.sh => ldmx-denv.bash} | 0 2 files changed, 3 insertions(+), 1 deletion(-) rename scripts/{ldmx-denv-transition.sh => ldmx-denv.bash} (100%) diff --git a/scripts/README.md b/scripts/README.md index 479d4312c..2dd47f18d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -32,6 +32,9 @@ help anyone interested in developing a wrapping script to be used within the shell of their choice. ### Initialization +In order to properly mimic the environment as defined by the `ldmx` +suite of bash functions, we need to initialize a denv with a few +options. ``` denv \ init \ # use denv special sub-command for initialization @@ -81,4 +84,3 @@ v0.0.0: just an idea - `mount` : pass arguments to `denv config mounts` - `setenv` : pass arguments to `denv config env copy` - `` : pass directly to `denv` - diff --git a/scripts/ldmx-denv-transition.sh b/scripts/ldmx-denv.bash similarity index 100% rename from scripts/ldmx-denv-transition.sh rename to scripts/ldmx-denv.bash From 4fbaa7b799c66b76cf4f30d8578705d0d42ef20d Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 10:52:19 -0500 Subject: [PATCH 4/9] copy workspace deduction into bash-specific setup script remove impossible POSIX-sh setup script and instead encourage users to wrap denv specifically for their shell or use denv directly --- scripts/ldmx-denv-init.sh | 126 -------------------------------------- scripts/ldmx-denv.bash | 102 ++++++++++++++++++++++++++++-- 2 files changed, 98 insertions(+), 130 deletions(-) delete mode 100644 scripts/ldmx-denv-init.sh diff --git a/scripts/ldmx-denv-init.sh b/scripts/ldmx-denv-init.sh deleted file mode 100644 index afd921386..000000000 --- a/scripts/ldmx-denv-init.sh +++ /dev/null @@ -1,126 +0,0 @@ -############################################################################### -# ldmx-denv-init.sh -# Make sure that the LDMX denv is initialized properly before proceeding. -# -# This script is desigend to be idempotent: i.e. if the denv is already -# properly configured, then this script should not have any effect. -############################################################################### - -# print each argument on its own line with the first line -# prefixed with "ERROR [ldmx-denv-init.sh]: ". -_error() { - printf >&2 "\033[1;31mERROR [ldmx-denv-init.sh]: \033[0m\033[31m%s\n" "$1" - shift - while [ "$#" -gt "0" ]; do - printf >&2 " %s\n" "$1" - shift - done - printf >&2 "\033[0m" -} - -# print the question passed to us and process user input -# continues in infinite loop if user is not explicitly answering yes or no -# Arguments -# 1 - question to ask the user (yes or no) -# Output -# returns 0 if user answers yes and 1 if user answers no -_user_confirm() { - question="$*" - while true; do - printf "\033[1m%s\033[0m [Y/n] " "${question}" - read -r ans - case "${ans}" in - Y|y|yes) - return 0 - ;; - N|n|no) - return 1 - ;; - *) - _error "${ans} is not one of Y, y, yes, N, n, or no." - ;; - esac - done -} - -if ! command -v denv > /dev/null 2>&1; then - _error "'denv' is not installed." - if _user_confirm "Do you wish to try to install 'denv' now?"; then - curl -s https://raw.githubusercontent.com/tomeichlersmith/denv/main/install | sh - else - printf "Not attempting to install 'denv'. Please follow the instructions at\n %s\n" \ - "https://tomeichlersmith.github.io/denv/getting_started.html#installation" - unset -f _error _user_confirm - return 1 - fi -fi - -if ! denv check --quiet; then - # maybe try loading apptainer via the 'module' command? - # https://github.com/LDMX-Software/ldmx-sw/issues/1248#issuecomment-1896618339 - denv check - _error "'denv' unable to find a supported container runner." \ - "Install one of the container runners 'denv' checked for above." - unset -f _error _user_confirm - return 2 -fi - -############################################################################### -# Attempt to deduce LDMX_BASE -# Unfortunately, deducing the path to a script being sourced is not possible -# using POSIX-compliant methods[1], so we need to fallback to separating -# different running scenarios. -# -# [1]: https://stackoverflow.com/a/29835459 -# -# Users can avoid all of this complication by pre-defining the LDMX_BASE -# environment variable. -############################################################################### -if [ -z "${LDMX_BASE+x}" ]; then - _full_path() ( - CDPATH= - cd -- "${1}" && pwd -P - ) - _base_from_script() { - _full_path "$(dirname -- "${1}")/../../" - } - if [ -n "${ZSH_VERSION}" ]; then - # zsh defines ZSH_EVAL_CONTEXT holding the details on how - # this code is being executed - # disable warnings about undefined variable and string indexing - # shellcheck disable=SC2154,SC3057 - LDMX_BASE="$(_base_from_script "${ZSH_EVAL_CONTEXT:file}")" - elif [ -n "${KSH_VERSION}" ]; then - # Korn defines the _ variable to the the file being sourced - # disable warning about undefined '_' - # shellcheck disable=SC3028 - LDMX_BASE="$(_base_from_script "${_}")" - elif [ -n "${BASH_VERSION}" ]; then - # running from bash, bash defines the BASH_SOURCE array to help us - # find the location of this file - # disable warning about undefined variable and array references - # shellcheck disable=SC3028,SC3054 - LDMX_BASE="$(_base_from_script "${BASH_SOURCE[0]}")" - else - # unable to deduce shell, resort to pwd - if expr "${PWD}" : ".*ldmx-sw$"; then - LDMX_BASE="$(_full_path ..)" - elif expr "${PWD}" : ".*ldmx-sw/scripts$"; then - LDMX_BASE="$(_full_path ../..)" - fi - fi - unset -f _full_path _base_from_script -fi -# re-export LDMX_BASE in case of user does inline variable definition like -# LDMX_BASE=/path/to/ldmx/base source /full/path/to/ldmx-denv-init.sh -export LDMX_BASE - -############################################################################### -# Check if the LDMX denv is initialized. If not, do a default initialization. -# TODO: update to `denv check --workspace` introduced in denv v0.7.0 to avoid -# reliance on internal implementation on how denv stores its config -############################################################################### -if ! denv check --workspace --quiet; then - denv init --clean-env --name "ldmx" "ldmx/dev:latest" "${LDMX_BASE}" -fi -unset -f _error _user_confirm diff --git a/scripts/ldmx-denv.bash b/scripts/ldmx-denv.bash index 727e4f396..885812d55 100644 --- a/scripts/ldmx-denv.bash +++ b/scripts/ldmx-denv.bash @@ -1,17 +1,110 @@ ############################################################################### -# ldmx-denv-transition.sh +# ldmx-env.bash # Replicate some of the `ldmx` commands by simply wrapping the equivalent # `denv` commands along with printouts explaining what is being done. # so that users know that the same procedures can be done with `denv`. ############################################################################### -# make sure denv is initialized -. "$(dirname "${BASH_SOURCE[0]}")/ldmx-denv-init.sh" +# print each argument on its own line with the first line +# prefixed with "ERROR [ldmx-denv.bash]: ". +_error() { + printf >&2 "\033[1;31mERROR [ldmx-denv.bash]: \033[0m\033[31m%s\n" "$1" + shift + while [ "$#" -gt "0" ]; do + printf >&2 " %s\n" "$1" + shift + done + printf >&2 "\033[0m" +} + +# print the question passed to us and process user input +# continues in infinite loop if user is not explicitly answering yes or no +# Arguments +# 1 - question to ask the user (yes or no) +# Output +# returns 0 if user answers yes and 1 if user answers no +_user_confirm() { + question="$*" + while true; do + printf "\033[1m%s\033[0m [Y/n] " "${question}" + read -r ans + case "${ans}" in + Y|y|yes) + return 0 + ;; + N|n|no) + return 1 + ;; + *) + _error "${ans} is not one of Y, y, yes, N, n, or no." + ;; + esac + done +} + +if ! command -v denv > /dev/null 2>&1; then + _error "'denv' is not installed." + if _user_confirm "Do you wish to try to install 'denv' now?"; then + curl -s https://raw.githubusercontent.com/tomeichlersmith/denv/main/install | sh + else + printf "Not attempting to install 'denv'. Please follow the instructions at\n %s\n" \ + "https://tomeichlersmith.github.io/denv/getting_started.html#installation" + unset -f _error _user_confirm + return 1 + fi +fi + +if ! denv check --quiet; then + # maybe try loading apptainer via the 'module' command? + # https://github.com/LDMX-Software/ldmx-sw/issues/1248#issuecomment-1896618339 + denv check + _error "'denv' unable to find a supported container runner." \ + "Install one of the container runners 'denv' checked for above." + unset -f _error _user_confirm + return 2 +fi + +############################################################################### +# Attempt to deduce LDMX_BASE +# Unfortunately, deducing the path to a script being sourced is not possible +# using POSIX-compliant methods[1], so we need to fallback to separating +# different running scenarios. +# +# [1]: https://stackoverflow.com/a/29835459 +# +# Users can avoid all of this complication by pre-defining the LDMX_BASE +# environment variable. +############################################################################### +if [ -z "${LDMX_BASE+x}" ]; then + _full_path() ( + CDPATH= + cd -- "${1}" && pwd -P + ) + _base_from_script() { + _full_path "$(dirname -- "${1}")/../../" + } + # running from bash, bash defines the BASH_SOURCE array to help us + # find the location of this file + # disable warning about undefined variable and array references + LDMX_BASE="$(_base_from_script "${BASH_SOURCE[0]}")" + unset -f _full_path _base_from_script +fi +# re-export LDMX_BASE in case of user does inline variable definition like +# LDMX_BASE=/path/to/ldmx/base source /full/path/to/ldmx-env.bash +export LDMX_BASE + +############################################################################### +# Check if the LDMX denv is initialized. If not, do a default initialization. +# This requires denv v0.7.0. +############################################################################### +if ! denv check --workspace --quiet; then + denv init --clean-env --name "ldmx" "ldmx/dev:latest" "${LDMX_BASE}" +fi # mimicing the ldmx bash function requires bash, point out user # doesn't need bash anymore if they use denv directly if [[ -z ${BASH} ]]; then - echo "[ldmx-env.sh] [ERROR] You aren't in a bash shell. You are in '$0'." + _error "You aren't in a bash shell. You are in '$0'." [[ "${SHELL}" = *"bash"* ]] || echo " Your default shell '${SHELL}' isn't bash." cat <<\HELP If you'd prefer to not use bash, you can use 'denv' directly rather than @@ -324,3 +417,4 @@ __ldmx_complete() { # Tell bash the tab-complete options for our main function ldmx complete -F __ldmx_complete ldmx +unset -f _error _user_confirm From a20c38cfa6bd20ce46aaa0ecfc834b4080d55a4c Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 11:00:51 -0500 Subject: [PATCH 5/9] add compile and recompFire to ldmx CLI spec --- scripts/README.md | 2 ++ scripts/ldmx-denv.bash | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 2dd47f18d..e1d687ff0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -83,4 +83,6 @@ v0.0.0: just an idea - `pull` : call `ldmx use` and then `denv config image pull` - `mount` : pass arguments to `denv config mounts` - `setenv` : pass arguments to `denv config env copy` + - `compile` : pass arguments to `denv ${LDMX_BASE}/ldmx-sw/ldmx-compile.sh` + - `recompFire` : pass arguments to `denv ${LDMX_BASE}/ldmx-sw/ldmx-recompFire.sh` - `` : pass directly to `denv` diff --git a/scripts/ldmx-denv.bash b/scripts/ldmx-denv.bash index 885812d55..49db035f7 100644 --- a/scripts/ldmx-denv.bash +++ b/scripts/ldmx-denv.bash @@ -10,7 +10,7 @@ _error() { printf >&2 "\033[1;31mERROR [ldmx-denv.bash]: \033[0m\033[31m%s\n" "$1" shift - while [ "$#" -gt "0" ]; do + while [[ "$#" -gt "0" ]]; do printf >&2 " %s\n" "$1" shift done @@ -75,7 +75,7 @@ fi # Users can avoid all of this complication by pre-defining the LDMX_BASE # environment variable. ############################################################################### -if [ -z "${LDMX_BASE+x}" ]; then +if [[ -z "${LDMX_BASE+x}" ]]; then _full_path() ( CDPATH= cd -- "${1}" && pwd -P @@ -176,6 +176,20 @@ __ldmx_setenv() { return $? } +__ldmx_compile() { + echo "denv ${LDMX_BASE}/ldmx-sw/scripts/ldmx-compile.sh $*" + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + denv "${LDMX_BASE}/ldmx-sw/scripts/ldmx-compile.sh" $@ +} + +__ldmx_recompFire() { + echo "denv ${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompFire.sh $*" + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + denv "${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompFire.sh" $@ +} + ############################################################################### # __ldmx_help # Print some helpful message to the terminal @@ -197,8 +211,12 @@ __ldmx_help() { ldmx pull (dev | pro | local) mount : Attach the input directory to the container when running ldmx mount - setenv : Set an environment variable in the container when running + setenv : Set an environment variable in the container when running ldmx setenv + compile : Configure and compile ldmx-sw + ldmx compile + recompFire : re-compile ldmx-sw and run fire with the input arguments + ldmx recompFire config.py : Run the input command in your current directory in the container ldmx [ ...] ldmx cmake .. @@ -263,6 +281,14 @@ ldmx() { __ldmx_use "$2" "$3" return $? ;; + compile|recompFire) + cmd="__ldmx_${1}" + shift + # intentionally re-splitting elements of an array + #shellcheck disable=SC2068 + ${cmd} $@ + return $? + ;; *) echo "denv $*" # intentionally re-splitting elements of an array @@ -366,7 +392,7 @@ __ldmx_complete() { if [[ "${COMP_CWORD}" = "1" ]]; then # tab completing a main argument - __ldmx_complete_command "config pull use run mount setenv" + __ldmx_complete_command "config pull use run mount setenv compile recompFire" elif [[ "${COMP_CWORD}" = "2" ]]; then # tab complete a sub-argument, # depends on the main argument From 2ad14937b475b04cb2b8cfd6961f156746dafaf3 Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 11:08:07 -0500 Subject: [PATCH 6/9] fix typo in path to recompile and fire script --- scripts/ldmx-denv.bash | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/ldmx-denv.bash b/scripts/ldmx-denv.bash index 49db035f7..354ab484c 100644 --- a/scripts/ldmx-denv.bash +++ b/scripts/ldmx-denv.bash @@ -184,10 +184,10 @@ __ldmx_compile() { } __ldmx_recompFire() { - echo "denv ${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompFire.sh $*" + echo "denv ${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompileAndFire.sh $*" # intentionally re-splitting elements of an array #shellcheck disable=SC2068 - denv "${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompFire.sh" $@ + denv "${LDMX_BASE}/ldmx-sw/scripts/ldmx-recompileAndFire.sh" $@ } ############################################################################### From 3ecf6fd282a68fc9e1105e4d8631d527f9e05a5e Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 11:08:20 -0500 Subject: [PATCH 7/9] shellcheck audit of compilation scripts avoid code repetition by calling compile from within recompile and fire --- scripts/ldmx-compile.sh | 17 ++++++++++++----- scripts/ldmx-recompileAndFire.sh | 22 ++++++++++------------ 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/scripts/ldmx-compile.sh b/scripts/ldmx-compile.sh index 57e7f7751..b15a9e97a 100755 --- a/scripts/ldmx-compile.sh +++ b/scripts/ldmx-compile.sh @@ -8,16 +8,23 @@ # or # ldmx setenv LDMX_COMPILE_BUILD= -if [ -z "$LDMX_COMPILE_CORES" ]; then +if [[ -z "${LDMX_COMPILE_CORES}" ]]; then LDMX_COMPILE_CORES=$(nproc) fi -if [ -z "$LDMX_COMPILE_BUILD" ]; then - LDMX_COMPILE_BUILD=${LDMX_BASE}/ldmx-sw/ +if [[ -z "${LDMX_COMPILE_BUILD}" ]]; then + if [[ -z "${LDMX_BASE}" ]]; then + echo "ERROR: LDMX_BASE is undefined." \ + " This script should be run within the containerized environment." + exit 1 + fi + LDMX_COMPILE_BUILD="${LDMX_BASE}/ldmx-sw" fi echo "-- Compiling ldmx-sw in ${LDMX_COMPILE_BUILD} with ${LDMX_COMPILE_CORES} cores" # Compile ldmx-sw -cmake -B ${LDMX_COMPILE_BUILD}/build -S ${LDMX_COMPILE_BUILD} $@ -cmake --build ${LDMX_COMPILE_BUILD}/build --target install -j=${LDMX_COMPILE_CORES} +# any arguments are passed to cmake to configure the build +# shellcheck disable=SC2068 +cmake -B "${LDMX_COMPILE_BUILD}/build" -S "${LDMX_COMPILE_BUILD}" $@ +cmake --build "${LDMX_COMPILE_BUILD}/build" --target install -j="${LDMX_COMPILE_CORES}" diff --git a/scripts/ldmx-recompileAndFire.sh b/scripts/ldmx-recompileAndFire.sh index 573275265..c01a7c936 100755 --- a/scripts/ldmx-recompileAndFire.sh +++ b/scripts/ldmx-recompileAndFire.sh @@ -8,19 +8,17 @@ # or # ldmx setenv LDMX_COMPILE_BUILD= - -if [ -z "$LDMX_COMPILE_CORES" ]; then - LDMX_COMPILE_CORES=$(nproc) -fi - -if [ -z "$LDMX_COMPILE_BUILD" ]; then - LDMX_COMPILE_BUILD=${LDMX_BASE}/ldmx-sw/ +if [[ -z "${LDMX_COMPILE_BUILD}" ]]; then + if [[ -z "${LDMX_BASE}" ]]; then + echo "ERROR: LDMX_BASE is undefined." \ + " This script should be run within the containerized environment." + exit 1 + fi + LDMX_COMPILE_BUILD="${LDMX_BASE}/ldmx-sw" fi -echo "-- Compiling ldmx-sw in ${LDMX_COMPILE_BUILD} with ${LDMX_COMPILE_CORES} cores, and running " - -# Compile ldmx-sw -cmake -B ${LDMX_COMPILE_BUILD}/build -S ${LDMX_COMPILE_BUILD} -cmake --build ${LDMX_COMPILE_BUILD}/build --target install -j=${LDMX_COMPILE_CORES} +# compile with default arguments +"${LDMX_COMPILE_BUILD}"/scripts/ldmx-compile.sh # Run fire on the input config +# shellcheck disable=SC2068 fire $@ From 19a8893b1d90ee6c84b11e385f4aa3dd2ab2d137 Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 11:14:33 -0500 Subject: [PATCH 8/9] sketch out some code for deducing CVMFS and legacy copy of images --- scripts/ldmx-denv.bash | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/scripts/ldmx-denv.bash b/scripts/ldmx-denv.bash index 354ab484c..35dbb26bb 100644 --- a/scripts/ldmx-denv.bash +++ b/scripts/ldmx-denv.bash @@ -128,16 +128,19 @@ fi __ldmx_use() { local _repo_name="$1" local _image_tag="$2" - cmd="denv config image ldmx/${_repo_name}:${_image_tag}" + local image="ldmx/${_repo_tag}:${_image_tag}" + local cvmfs_base="/cvmfs/unpacked.cern.ch/registry.hub.docker.com/" + if [[ -e "${cvmfs_base}/${image}" ]]; then + image="${cvmfs_base}/${image}" + elif [[ -e "${LDMX_BASE}/ldmx_${_repo_name}_${image_tag}.sif" ]]; then + image="${LDMX_BASE}/ldmx_${_repo_name}_${image_tag}.sif" + fi + cmd="denv config image ${image}" echo "${cmd}" ${cmd} return $? } -############################################################################### -# __ldmx_config -# Print the configuration of the current setup -############################################################################### __ldmx_config() { echo "LDMX base directory: ${LDMX_BASE}" echo "uname: $(uname -a)" @@ -147,12 +150,6 @@ __ldmx_config() { return $? } -############################################################################### -# __ldmx_mount -# Tell us to mount the passed directory to the container when we run -# By default, we already mount the LDMX_BASE directory, so none of -# its subdirectories need to (or should be) specified. -############################################################################### __ldmx_mount() { echo "denv config mounts $*" # intentionally re-splitting elements of an array @@ -161,13 +158,6 @@ __ldmx_mount() { return $? } -############################################################################### -# __ldmx_setenv -# Tell us to pass an environment variable to the container when we run -# By default, we pass the LDMX_BASE and DISPLAY variables explicitly, -# because their syntax is too different between docker and singularity, -# so none of these need to (or should be) specified. -############################################################################### __ldmx_setenv() { echo "denv config env copy $*" # intentionally re-splitting elements of an array From 5bfd8b77c3cc9e7e5b107e3b5d03779de4d5884a Mon Sep 17 00:00:00 2001 From: tomeichlersmith Date: Tue, 21 May 2024 11:28:17 -0500 Subject: [PATCH 9/9] fix some typos in ldmx and ldmx use --- scripts/ldmx-denv.bash | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/ldmx-denv.bash b/scripts/ldmx-denv.bash index 35dbb26bb..e33d3cadf 100644 --- a/scripts/ldmx-denv.bash +++ b/scripts/ldmx-denv.bash @@ -128,7 +128,7 @@ fi __ldmx_use() { local _repo_name="$1" local _image_tag="$2" - local image="ldmx/${_repo_tag}:${_image_tag}" + local image="ldmx/${_repo_name}:${_image_tag}" local cvmfs_base="/cvmfs/unpacked.cern.ch/registry.hub.docker.com/" if [[ -e "${cvmfs_base}/${image}" ]]; then image="${cvmfs_base}/${image}" @@ -272,7 +272,6 @@ ldmx() { return $? ;; compile|recompFire) - cmd="__ldmx_${1}" shift # intentionally re-splitting elements of an array #shellcheck disable=SC2068