Add dev nvidia with nv

CHANGELOG.md

@@ -12,6 +12,9 @@ and changes prior to that are (unfortunately) done retrospectively. Critical ite
  - migration guidance (how to convert images?)
  - changed behaviour (recipe sections work differently)
 
+## [v2.4.5](https://github.com/singularityware/singularity/tree/release-2.5)
+ - Fix conflict between `--nv` and `--contain` options
+
 ## [v2.4.4](https://github.com/singularityware/singularity/tree/release-2.4)
 
  - Removed capability to handle docker layer aufs whiteout files correctly as

libexec/cli/action_argparser.sh

@@ -123,6 +123,11 @@ while true; do
         ;;
         --nv)
             shift
+            # if the --contain option is used, we will also need to bind mount
+            # whatever nvidia devices exist. $SINGULARITY_NVDEV is used in 
+            # src/lib/runtime/mounts/dev/dev.c
+            SINGULARITY_NVDEV=`ls /dev/nvidia* | tr '\n' ','`
+            export SINGULARITY_NV SINGULARITY_NVDEV
             SINGULARITY_NVLIBLIST=`mktemp ${TMPDIR:-/tmp}/.singularity-nvliblist.XXXXXXXX`
             cat $SINGULARITY_sysconfdir"/singularity/nvliblist.conf" | grep -Ev "^#|^\s*$" > $SINGULARITY_NVLIBLIST
             for i in $(ldconfig -p | grep -f "${SINGULARITY_NVLIBLIST}"); do
@@ -141,12 +146,14 @@ while true; do
             else
                 export SINGULARITY_CONTAINLIBS
             fi
-            if NVIDIA_SMI=$(which nvidia-smi); then
+            if NVIDIA_SMI=$(which nvidia-smi >/dev/null 2>&1); then
                 if [ -n "${SINGULARITY_BINDPATH:-}" ]; then
                     SINGULARITY_BINDPATH="${SINGULARITY_BINDPATH},${NVIDIA_SMI}"
                 else
                     SINGULARITY_BINDPATH="${NVIDIA_SMI}"
                 fi
+
+                # SINGULARITY_BINDPATH="${SINGULARITY_BINDPATH},/dev/nvidia0"
                 export SINGULARITY_BINDPATH
             else
                 message WARN "Could not find the Nvidia SMI binary to bind into container\n"

src/lib/runtime/mounts/dev/dev.c

@@ -31,6 +31,7 @@
 #include <unistd.h>
 #include <stdlib.h>
 #include <grp.h>
+#include <string.h>
 
 #include "config.h"
 #include "util/file.h"
@@ -52,6 +53,7 @@ int _singularity_runtime_mount_dev(void) {
     if ( ( singularity_registry_get("CONTAIN") != NULL ) || ( strcmp("minimal", singularity_config_get_value(MOUNT_DEV)) == 0 ) ) {
         char *sessiondir = singularity_registry_get("SESSIONDIR");
         char *devdir = joinpath(sessiondir, "/dev");
+        char *nvdevs = singularity_registry_get("NVDEV"); 
 
         if ( is_dir(joinpath(container_dir, "/dev")) < 0 ) {
             int ret;
@@ -103,6 +105,17 @@ int _singularity_runtime_mount_dev(void) {
         bind_dev(sessiondir, "/dev/random");
         bind_dev(sessiondir, "/dev/urandom");
 
+        /* if the user passed the --nv flag and the --contain flag, still bind
+        nvidia devices */
+        if ( nvdevs != NULL ) {
+            char *nvdev = strtok(nvdevs, ",");
+            while ( nvdev != NULL ) {
+                singularity_message(2, "Binding device %s\n", nvdev);
+                bind_dev(sessiondir, nvdev);
+                nvdev = strtok(NULL, ",");
+            }
+        }
+
         singularity_message(DEBUG, "Mounting tmpfs for staged /dev/shm\n");
         if ( singularity_mount("/dev/shm", joinpath(devdir, "/shm"), "tmpfs", MS_NOSUID, "") < 0 ) {
             singularity_message(ERROR, "Failed to mount %s: %s\n", joinpath(devdir, "/shm"), strerror(errno));