Update singularity support

opal/mca/shmem/mmap/shmem_mmap_component.c

@@ -176,13 +176,6 @@ mmap_open(void)
 static int
 mmap_query(mca_base_module_t **module, int *priority)
 {
-    /* if we are in a container, then we must disqualify ourselves */
-    if (NULL != getenv("OPAL_PROC_CONTAINER")) {
-        *priority = 0;
-        *module = NULL;
-        return OPAL_ERROR;
-    }
-
     *priority = mca_shmem_mmap_component.priority;
     *module = (mca_base_module_t *)&opal_shmem_mmap_module.super;
     return OPAL_SUCCESS;

opal/mca/shmem/posix/shmem_posix_component.c

@@ -201,13 +201,6 @@ posix_runtime_query(mca_base_module_t **module,
 static int
 posix_query(mca_base_module_t **module, int *priority)
 {
-    /* if we are in a container, then we must disqualify ourselves */
-    if (NULL != getenv("OPAL_PROC_CONTAINER")) {
-        *priority = 0;
-        *module = NULL;
-        return OPAL_ERROR;
-    }
-
     *priority = mca_shmem_posix_component.priority;
     *module = (mca_base_module_t *)&opal_shmem_posix_module.super;
     return OPAL_SUCCESS;

opal/mca/shmem/sysv/shmem_sysv_component.c

@@ -210,13 +210,6 @@ sysv_runtime_query(mca_base_module_t **module, int *priority, const char *hint)
 static int
 sysv_query(mca_base_module_t **module, int *priority)
 {
-    /* if we are in a container, then we must disqualify ourselves */
-    if (NULL != getenv("OPAL_PROC_CONTAINER")) {
-        *priority = 0;
-        *module = NULL;
-        return OPAL_ERROR;
-    }
-
     *priority = mca_shmem_sysv_component.priority;
     *module = (mca_base_module_t *)&opal_shmem_sysv_module.super;
     return OPAL_SUCCESS;

orte/mca/schizo/alps/schizo_alps.c

@@ -29,9 +29,11 @@
 #include "schizo_alps.h"
 
 static orte_schizo_launch_environ_t check_launch_environment(void);
+static void finalize(void);
 
 orte_schizo_base_module_t orte_schizo_alps_module = {
-    .check_launch_environment = check_launch_environment
+    .check_launch_environment = check_launch_environment,
+    .finalize = finalize
 };
 
 static char **pushed_envs = NULL;

orte/mca/schizo/base/base.h

@@ -68,6 +68,8 @@ ORTE_DECLSPEC int orte_schizo_base_parse_env(char **personality,
                                              opal_cmd_line_t *cmd_line,
                                              char **srcenv,
                                              char ***dstenv);
+ORTE_DECLSPEC int orte_schizo_base_setup_app(char **personality,
+                                             orte_app_context_t *app);
 ORTE_DECLSPEC int orte_schizo_base_setup_fork(orte_job_t *jdata,
                                               orte_app_context_t *context);
 ORTE_DECLSPEC int orte_schizo_base_setup_child(orte_job_t *jobdat,

orte/mca/schizo/base/schizo_base_frame.c

@@ -39,6 +39,7 @@ orte_schizo_base_t orte_schizo_base = {{{0}}};
 orte_schizo_base_module_t orte_schizo = {
     .parse_cli = orte_schizo_base_parse_cli,
     .parse_env = orte_schizo_base_parse_env,
+    .setup_app = orte_schizo_base_setup_app,
     .setup_fork = orte_schizo_base_setup_fork,
     .setup_child = orte_schizo_base_setup_child,
     .check_launch_environment = orte_schizo_base_check_launch_environment,

orte/mca/schizo/base/schizo_base_stubs.c

@@ -19,8 +19,8 @@
 #include "orte/util/name_fns.h"
 #include "orte/mca/schizo/base/base.h"
 
-const char* orte_schizo_base_print_env(orte_schizo_launch_environ_t env)
-{
+ const char* orte_schizo_base_print_env(orte_schizo_launch_environ_t env)
+ {
     switch(env) {
         case ORTE_SCHIZO_UNDETERMINED:
             return "UNDETERMINED";
@@ -80,6 +80,24 @@ int orte_schizo_base_parse_env(char **personality,
     return ORTE_SUCCESS;
 }
 
+int orte_schizo_base_setup_app(char **personality,
+                               orte_app_context_t *app)
+{
+    int rc;
+    orte_schizo_base_active_module_t *mod;
+
+    OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) {
+        if (NULL != mod->module->setup_app) {
+            rc = mod->module->setup_app(personality, app);
+            if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+        }
+    }
+    return ORTE_SUCCESS;
+}
+
 int orte_schizo_base_setup_fork(orte_job_t *jdata,
                                 orte_app_context_t *context)
 {

orte/mca/schizo/schizo.h

@@ -40,21 +40,44 @@ BEGIN_C_DECLS
 * the base stub functions
 */
 
+/* initialize the module - allow it to do whatever one-time
+ * things it requires */
 typedef int (*orte_schizo_base_module_init_fn_t)(void);
 
+/* given an argv-array of personalities, parse a tool command line
+ * starting from the given location according to the cmd line options
+ * known to this module's personality. First, of course, check that
+ * this module is included in the specified array of personalities!
+ * Only one command-line parser is allowed to operate - i.e., if */
 typedef int (*orte_schizo_base_module_parse_cli_fn_t)(char **personality,
                                                       int argc, int start,
                                                       char **argv);
 
+/* given an argv-array of personalities, parse the environment of the
+ * tool to extract any personality-specific envars that need to be
+ * forward to the app's environment upon execution */
 typedef int (*orte_schizo_base_module_parse_env_fn_t)(char **personality,
                                                       char *path,
                                                       opal_cmd_line_t *cmd_line,
                                                       char **srcenv,
                                                       char ***dstenv);
 
+/* given an argv-array of personalities, do whatever preparation work
+ * is required to setup the app for execution. This is intended to be
+ * used by orterun and other launcher tools to, for example, change
+ * an executable's relative-path to an absolute-path, or add a command
+ * required for starting a particular kind of application (e.g., adding
+ * "java" to start a Java application) */
+typedef int (*orte_schizo_base_module_setup_app_fn_t)(char **personality,
+                                                      orte_app_context_t *app);
+
+/* add any personality-specific envars required at the job level prior
+ * to beginning to execute local procs */
 typedef int (*orte_schizo_base_module_setup_fork_fn_t)(orte_job_t *jdata,
                                                        orte_app_context_t *context);
 
+/* add any personality-specific envars required for this specific local
+ * proc upon execution */
 typedef int (*orte_schizo_base_module_setup_child_fn_t)(orte_job_t *jdata,
                                                         orte_proc_t *child,
                                                         orte_app_context_t *app);
@@ -86,6 +109,7 @@ typedef struct {
     orte_schizo_base_module_init_fn_t                   init;
     orte_schizo_base_module_parse_cli_fn_t              parse_cli;
     orte_schizo_base_module_parse_env_fn_t              parse_env;
+    orte_schizo_base_module_setup_app_fn_t              setup_app;
     orte_schizo_base_module_setup_fork_fn_t             setup_fork;
     orte_schizo_base_module_setup_child_fn_t            setup_child;
     orte_schizo_base_module_ck_launch_environ_fn_t      check_launch_environment;

orte/mca/schizo/singularity/schizo_singularity.c

@@ -20,31 +20,36 @@
 #include "opal/util/argv.h"
 #include "opal/util/basename.h"
 #include "opal/util/opal_environ.h"
+#include "opal/util/os_dirpath.h"
+#include "opal/util/path.h"
 
 #include "orte/runtime/orte_globals.h"
 #include "orte/util/name_fns.h"
 #include "orte/mca/schizo/base/base.h"
 
 #include "schizo_singularity.h"
 
+static int setup_app(char **personality,
+                     orte_app_context_t *context);
 static int setup_fork(orte_job_t *jdata,
                       orte_app_context_t *context);
 
 orte_schizo_base_module_t orte_schizo_singularity_module = {
+    .setup_app = setup_app,
     .setup_fork = setup_fork
 };
 
-static int setup_fork(orte_job_t *jdata,
-                      orte_app_context_t *app)
+static int setup_app(char **personality,
+                     orte_app_context_t *app)
 {
     int i;
-    char *newenv;
+    char *newenv, *pth;
     bool takeus = false;
-    char *cmd, *tmp = NULL, *p, *t2;
+    char *p, *t2;
 
     /* see if we are included */
-    for (i=0; NULL != jdata->personality[i]; i++) {
-        if (0 == strcmp(jdata->personality[i], "singularity")) {
+    for (i=0; NULL != personality[i]; i++) {
+        if (0 == strcmp(personality[i], "singularity")) {
             takeus = true;
             break;
         }
@@ -64,55 +69,104 @@ static int setup_fork(orte_job_t *jdata,
                         "%s schizo:singularity: checking app %s",
                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[0]);
 
+    if (0 < strlen(OPAL_SINGULARITY_PATH)) {
+        asprintf(&pth, "%s/singularity", OPAL_SINGULARITY_PATH);
+    } else {
+        /* since we allow for detecting singularity's presence, it
+         * is possible that we found it in the PATH, but not in a
+         * standard location. Check for that here */
+         pth = opal_path_findv("singularity", X_OK, app->env, NULL);
+         if (NULL == pth) {
+            /* cannot execute */
+            return ORTE_ERR_TAKE_NEXT_OPTION;
+         }
+    }
     /* find the path and prepend it with the path to Singularity */
     for (i = 0; NULL != app->env && NULL != app->env[i]; ++i) {
         /* add to PATH */
         if (0 == strncmp("PATH=", app->env[i], 5)) {
-            asprintf(&newenv, "%s:%s", OPAL_SINGULARITY_PATH, app->env[i] + 5);
+            t2 = opal_dirname(pth);
+            asprintf(&newenv, "%s:%s", t2, app->env[i] + 5);
             opal_setenv("PATH", newenv, true, &app->env);
             free(newenv);
+            free(t2);
             break;
         }
     }
 
-    /* flag that the app is in a container */
-    opal_setenv("OPAL_PROC_CONTAINER", "1", true, &app->env);
-
     /* ensure that we use "singularity run" to execute this app */
     if (0 != strcmp(app->app, "singularity")) {
         opal_output_verbose(1, orte_schizo_base_framework.framework_output,
-                            "%s schizo:singularity: adding singularity cmds at %s",
-                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), OPAL_SINGULARITY_PATH);
+                            "%s schizo:singularity: adding singularity cmd %s",
+                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), pth);
         /* change the app to the "singularity" command */
         free(app->app);
-        if (0 < strlen(OPAL_SINGULARITY_PATH)) {
-            asprintf(&app->app, "%s/singularity", OPAL_SINGULARITY_PATH);
-        } else {
-            app->app = strdup("singularity");
-        }
-        /* if the app contains .sapp, then we need to strip that
-         * extension so singularity doesn't bark at us */
-        if (NULL != (p = strstr(app->argv[0], ".sapp"))) {
-            tmp = strdup(app->argv[0]);
-            t2 = opal_basename(app->argv[0]);
-            p = strstr(t2, ".sapp");
-            *p = '\0'; // strip the extension
-            free(app->argv[0]);
-            app->argv[0] = t2;
+        app->app = pth;
+    } else {
+        free(pth);
+    }
+
+    /* if the app contains .sapp, then we need to strip that
+     * extension so singularity doesn't bark at us */
+    if (NULL != (p = strstr(app->argv[0], ".sapp"))) {
+        t2 = opal_basename(app->argv[0]);
+        p = strstr(t2, ".sapp");
+        *p = '\0'; // strip the extension
+        free(app->argv[0]);
+        app->argv[0] = t2;
+    }
+    opal_argv_prepend_nosize(&app->argv, "run");
+    opal_argv_prepend_nosize(&app->argv, "singularity");
+
+    /* export an envar to permit shared memory operations */
+    opal_setenv("SINGULARITY_NO_NAMESPACE_PID", "1", true, &app->env);
+
+    return ORTE_SUCCESS;
+}
+
+static int setup_fork(orte_job_t *jdata,
+                      orte_app_context_t *app)
+{
+    int i;
+    bool takeus = false;
+    char *p;
+    char dir[MAXPATHLEN];
+
+    /* see if we are included */
+    for (i=0; NULL != jdata->personality[i]; i++) {
+        if (0 == strcmp(jdata->personality[i], "singularity")) {
+            takeus = true;
+            break;
         }
-        opal_argv_prepend_nosize(&app->argv, "run");
-        opal_argv_prepend_nosize(&app->argv, "singularity");
     }
-    /* ensure this application has been "installed" */
-    if (NULL != tmp) {
-        opal_output_verbose(1, orte_schizo_base_framework.framework_output,
-                            "%s schizo:singularity: installing container %s",
-                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), tmp);
-        (void)asprintf(&cmd, "singularity install %s >> /dev/null", tmp);
-        system(cmd);
-        free(cmd);
-        free(tmp);
+    if (!takeus) {
+        /* even if they didn't specify, check to see if
+         * this involves a singularity container */
+        if (0 != strcmp(app->argv[0],"singularity") &&
+            0 != strcmp(app->argv[0],"sapprun") &&
+            NULL == strstr(app->argv[0], ".sapp")) {
+            /* guess not! */
+            return ORTE_ERR_TAKE_NEXT_OPTION;
+        }
     }
 
+    /* save our current directory */
+    getcwd(dir, sizeof(dir));
+
+    /* change to the working directory for this context */
+    chdir(app->cwd);
+
+    /* ensure the app is installed */
+    opal_output_verbose(1, orte_schizo_base_framework.framework_output,
+                        "%s schizo:singularity: installing app %s",
+                        ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[2]);
+    (void)asprintf(&p, "%s install %s.sapp &> /dev/null", app->app, app->argv[2]);
+    system(p);
+    free(p);
+
+    /* return to the original directory */
+    chdir(dir);
+
     return ORTE_SUCCESS;
 }
+

orte/tools/orterun/orterun.c

@@ -1258,6 +1258,9 @@ static int parse_locals(orte_job_t *jdata, int argc, char* argv[])
                     ++app_num;
                     opal_pointer_array_add(jdata->apps, app);
                     ++jdata->num_apps;
+                    if (ORTE_SUCCESS != (rc = orte_schizo.setup_app(jdata->personality, app))) {
+                        return rc;
+                    }
                 }
 
                 /* Reset the temps */
@@ -1284,6 +1287,9 @@ static int parse_locals(orte_job_t *jdata, int argc, char* argv[])
             ++app_num;
             opal_pointer_array_add(jdata->apps, app);
             ++jdata->num_apps;
+            if (ORTE_SUCCESS != (rc = orte_schizo.setup_app(jdata->personality, app))) {
+                return rc;
+            }
         }
     }
     if (NULL != env) {
@@ -1670,6 +1676,8 @@ static int create_app(int argc, char* argv[],
         rc = ORTE_ERR_NOT_FOUND;
         goto cleanup;
     }
+    free(app->argv[0]);
+    app->argv[0] = opal_basename(app->app);
 
     /* if this is a Java application, we have a bit more work to do. Such
      * applications actually need to be run under the Java virtual machine

orte/util/context_fns.c

@@ -130,16 +130,18 @@ int orte_util_check_context_app(orte_app_context_t *context, char **env)
         bproc, for example, does not use the fork pls for launching, so
         it does this same work over there. */
 
-    tmp = opal_basename(context->argv[0]);
-    if (strlen(tmp) == strlen(context->argv[0])) {
+    tmp = opal_basename(context->app);
+    if (strlen(tmp) == strlen(context->app)) {
         /* If this is a naked executable -- no relative or absolute
         pathname -- then search the PATH for it */
         free(tmp);
-        tmp = opal_path_findv(context->argv[0], X_OK, env, context->cwd);
+        tmp = opal_path_findv(context->app, X_OK, env, context->cwd);
         if (NULL == tmp) {
             return ORTE_ERR_EXE_NOT_FOUND;
         }
-        if (NULL != context->app) free(context->app);
+        if (NULL != context->app) {
+            free(context->app);
+        }
         context->app = tmp;
     } else {
         free(tmp);