CROM-6914 Use singularity for HPC testing (#6736)

broadinstitute · May 13, 2024 · 9a3a8bf · 9a3a8bf
1 parent 430fc40
commit 9a3a8bf
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 3 deletions.
diff --git a/src/ci/bin/testCentaurSlurm.sh b/src/ci/bin/testCentaurSlurm.sh
@@ -5,18 +5,24 @@ set -o errexit -o nounset -o pipefail
 # shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
 # shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test_singularity.inc.sh" || source test_singularity.inc.sh
+# shellcheck source=/dev/null
 source "${BASH_SOURCE%/*}/test_slurm.inc.sh" || source test_slurm.inc.sh
 
 cromwell::build::setup_common_environment
 
 cromwell::build::slurm::setup_slurm_environment
 
+cromwell::build::singularity::setup_singularity_environment
+
 cromwell::build::setup_centaur_environment
 
 cromwell::build::assemble_jars
 
 cromwell::build::run_centaur \
     -e call_cache_capoeira_local \
     -e draft3_call_cache_capoeira_local \
+    -e non_root_default_user \
+    -e non_root_specified_user \
 
 cromwell::build::generate_code_coverage
diff --git a/src/ci/bin/test_singularity.inc.sh b/src/ci/bin/test_singularity.inc.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+set -o errexit -o nounset -o pipefail
+# import in shellcheck / CI / IntelliJ compatible ways
+# shellcheck source=/dev/null
+source "${BASH_SOURCE%/*}/test.inc.sh" || source test.inc.sh
+
+# A set of common singularity functions for use in other scripts.
+#
+# Functions:
+#
+#   - cromwell::build::singularity::*
+#     Functions for use in other Singularity scripts
+#
+#   - cromwell::private::singularity::*
+#     Functions for use only within this file by cromwell::build::singularity::* functions
+#
+
+cromwell::build::singularity::setup_singularity_environment() {
+    # Installs one of the singularity forks on Ubuntu
+    # https://old.reddit.com/r/HPC/comments/r61bto/singularity_joins_the_linux_foundation_and_is/
+
+    # Using https://apptainer.org/docs/admin/1.3/installation.html#install-ubuntu-packages
+    sudo apt install -y software-properties-common
+    sudo add-apt-repository -y ppa:apptainer/ppa
+    sudo apt update
+    sudo apt install -y apptainer
+}
diff --git a/src/ci/resources/slurm_application.conf b/src/ci/resources/slurm_application.conf
@@ -9,13 +9,29 @@ backend {
       config {
         runtime-attributes = """
           String? docker
-          String? docker_user
         """
 
         # https://slurm.schedmd.com/sbatch.html
         submit-docker = """
-        sbatch -J ${job_name} -D ${cwd} --wrap \
-          "/bin/bash -c 'docker run --rm ${"--user " + docker_user} --entrypoint ${job_shell} -v ${cwd}:${docker_cwd} ${docker} ${docker_script}'"
+            set -euo pipefail
+            CACHE_DIR=$HOME/.singularity/cache
+            mkdir -p $CACHE_DIR
+            LOCK_FILE=$CACHE_DIR/singularity_pull_flock
+            DOCKER_NAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< ${docker})
+            IMAGE=$CACHE_DIR/$DOCKER_NAME.sif
+            (
+              flock --verbose --exclusive --timeout 900 9 || exit 1
+              if [ ! -e "$IMAGE" ]; then
+                singularity build $IMAGE docker://${docker}
+              fi
+            ) 9>$LOCK_FILE
+
+            sbatch \
+              -J ${job_name} \
+              -D ${cwd} \
+              -o ${cwd}/execution/stdout \
+              -e ${cwd}/execution/stderr \
+              --wrap "singularity exec --containall --bind ${cwd}:${docker_cwd} $IMAGE ${job_shell} ${docker_script}"
         """
 
         # https://slurm.schedmd.com/scancel.html