Remove stale global variables

.gitignore

@@ -111,6 +111,7 @@ contrib/platform/intel/bend/*orcm*
 contrib/scaling/orte_no_op
 contrib/scaling/mpi_no_op
 contrib/scaling/mpi_barrier
+contrib/scaling/mpi_memprobe
 
 examples/hello_c
 examples/hello_cxx

contrib/scaling/Makefile

@@ -1,4 +1,4 @@
-PROGS = orte_no_op mpi_no_op
+PROGS = orte_no_op mpi_no_op mpi_memprobe
 
 all: $(PROGS)
 
@@ -10,5 +10,8 @@ orte_no_op:
 mpi_no_op:
 	mpicc -o mpi_no_op mpi_no_op.c
 
+mpi_memprobe:
+	mpicc -o mpi_memprobe mpi_memprobe.c -lopen-pal
+
 clean:
 	rm -f $(PROGS) *~

contrib/scaling/mpi_memprobe.c

@@ -0,0 +1,98 @@
+/* -*- C -*-
+ *
+ * $HEADER$
+ *
+ * The most basic of MPI applications
+ */
+
+#include "orte_config.h"
+
+#include <stdio.h>
+#include "mpi.h"
+#include "opal/mca/pmix/pmix.h"
+#include "orte/runtime/runtime.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/name_fns.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+static volatile int active;
+static volatile bool wait_for_release = true;
+#define MEMPROBE_RELEASE 12345
+
+static void _release_fn(int status,
+                        const opal_process_name_t *source,
+                        opal_list_t *info, opal_list_t *results,
+                        opal_pmix_notification_complete_fn_t cbfunc,
+                        void *cbdata)
+{
+    /* must let the notifier know we are done */
+    if (NULL != cbfunc) {
+        cbfunc(0, NULL, NULL, NULL, cbdata);
+    }
+    /* flag that the debugger is complete so we can exit */
+    wait_for_release = false;
+}
+
+static void _register_fn(int status,
+                         size_t evhandler_ref,
+                         void *cbdata)
+{
+    volatile int *active = (volatile int*)cbdata;
+
+    if (0 != status) {
+        fprintf(stderr, "Client EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n",
+                   status, (unsigned long)evhandler_ref);
+    }
+    *active = status;
+}
+
+int main(int argc, char* argv[])
+{
+    int rank, size;
+    opal_list_t *codes;
+    opal_value_t *kv;
+
+    MPI_Init(&argc, &argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    if (0 == rank) {
+        fprintf(stderr, "Sampling memory usage after MPI_Init\n");
+    }
+
+    codes = OBJ_NEW(opal_list_t);
+    kv = OBJ_NEW(opal_value_t);
+    kv->key = strdup("errorcode");
+    kv->type = OPAL_INT;
+    kv->data.integer = MEMPROBE_RELEASE;
+    opal_list_append(codes, &kv->super);
+
+    active = -1;
+    opal_pmix.register_evhandler(codes, NULL, _release_fn, _register_fn, (void*)&active);
+    while (-1 == active) {
+        usleep(10);
+    }
+
+    /* now wait for notification */
+    while (wait_for_release) {
+        usleep(10);
+    }
+    wait_for_release = true;
+
+    /* perform a barrier so some communication will occur, thus
+     * requiring exchange of endpoint info */
+    MPI_Barrier(MPI_COMM_WORLD);
+
+    if (0 == rank) {
+        fprintf(stderr, "\n\nSampling memory usage after MPI_Barrier\n");
+    }
+
+    /* wait again while memory is sampled */
+    while (wait_for_release) {
+        usleep(10);
+    }
+
+    MPI_Finalize();
+    return 0;
+}

opal/mca/pmix/pmix2x/pmix/include/pmix_common.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2013-2016 Intel, Inc. All rights reserved
+ * Copyright (c) 2013-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016      IBM Corporation.  All rights reserved.
@@ -157,6 +157,7 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_TDIR_RMCLEAN                   "pmix.tdir.rmclean"     // (bool)  Resource Manager will clean session directories
 
 /* information about relative ranks as assigned by the RM */
+#define PMIX_PROCID                         "pmix.procid"           // (pmix_proc_t) process identifier
 #define PMIX_NSPACE                         "pmix.nspace"           // (char*) nspace of a job
 #define PMIX_JOBID                          "pmix.jobid"            // (char*) jobid assigned by scheduler
 #define PMIX_APPNUM                         "pmix.appnum"           // (uint32_t) app number within the job
@@ -282,6 +283,8 @@ typedef uint32_t pmix_rank_t;
 #define PMIX_QUERY_AUTHORIZATIONS           "pmix.qry.auths"         // return operations tool is authorized to perform
 #define PMIX_QUERY_SPAWN_SUPPORT            "pmix.qry.spawn"         // return a comma-delimited list of supported spawn attributes
 #define PMIX_QUERY_DEBUG_SUPPORT            "pmix.qry.debug"         // return a comma-delimited list of supported debug attributes
+#define PMIX_QUERY_MEMORY_USAGE             "pmix.qry.mem"           // return info on memory usage for the procs indicated in the qualifiers
+#define PMIX_QUERY_LOCAL_ONLY               "pmix.qry.local"         // constrain the query to local information only
 
 /* log attributes */
 #define PMIX_LOG_STDERR                     "pmix.log.stderr"        // (bool) log data to stderr

opal/mca/pmix/pmix2x/pmix/src/client/pmix_client.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -553,11 +553,18 @@ static void _putfn(int sd, short args, void *cbdata)
 {
     pmix_cb_t *cb = (pmix_cb_t*)cbdata;
     pmix_status_t rc;
-    pmix_kval_t *kv;
+    pmix_kval_t *kv = NULL;
     pmix_nspace_t *ns;
     uint8_t *tmp;
     size_t len;
 
+    /* no need to push info that starts with "pmix" as that is
+     * info we would have been provided at startup */
+    if (0 == strncmp(cb->key, "pmix", 4)) {
+        rc = PMIX_SUCCESS;
+        goto done;
+    }
+
     /* setup to xfer the data */
     kv = PMIX_NEW(pmix_kval_t);
     kv->key = strdup(cb->key);  // need to copy as the input belongs to the user
@@ -622,7 +629,9 @@ static void _putfn(int sd, short args, void *cbdata)
     }
 
   done:
-    PMIX_RELEASE(kv);  // maintain accounting
+    if (NULL != kv) {
+        PMIX_RELEASE(kv);  // maintain accounting
+    }
     cb->pstatus = rc;
     cb->active = false;
 }

opal/mca/pmix/pmix2x/pmix/src/client/pmix_client_get.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014      Artem Y. Polyakov <artpol84@gmail.com>.
@@ -763,7 +763,7 @@ static void _getnbfn(int fd, short flags, void *cbdata)
      * us to attempt to retrieve it from the server */
     for (n=0; n < cb->ninfo; n++) {
         if (0 == strcmp(cb->info[n].key, PMIX_OPTIONAL) &&
-            cb->info[n].value.data.flag) {
+            (PMIX_UNDEF == cb->info[n].value.type || cb->info[n].value.data.flag)) {
             /* they don't want us to try and retrieve it */
             pmix_output_verbose(2, pmix_globals.debug_output,
                                 "PMIx_Get key=%s for rank = %d, namespace = %s was not found - request was optional",

opal/mca/pmix/pmix2x/pmix/src/event/pmix_event_registration.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2016 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -356,6 +356,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
         sing->code = cd->codes[0];
         index = pmix_globals.events.nhdlrs;
         sing->index = index;
+        sing->evhdlr = cd->evhdlr;
         ++pmix_globals.events.nhdlrs;
         sing->cbobject = cbobject;
         rc = _add_hdlr(&pmix_globals.events.single_events, &sing->super,
@@ -365,17 +366,17 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
             PMIX_ERR_WOULD_BLOCK != rc) {
                 /* unable to register */
             --pmix_globals.events.nhdlrs;
-        rc = PMIX_ERR_EVENT_REGISTRATION;
-        index = UINT_MAX;
+            rc = PMIX_ERR_EVENT_REGISTRATION;
+            index = UINT_MAX;
+            goto ack;
+        }
+        if (PMIX_ERR_WOULD_BLOCK == rc) {
+            /* the callback will provide our response */
+            PMIX_RELEASE(cd);
+            return;
+        }
         goto ack;
     }
-    if (PMIX_ERR_WOULD_BLOCK == rc) {
-        /* the callback will provide our response */
-        PMIX_RELEASE(cd);
-        return;
-    }
-    goto ack;
-    }
 
     /* must be a multi-code registration */
     multi = PMIX_NEW(pmix_multi_event_t);
@@ -387,6 +388,7 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
     memcpy(multi->codes, cd->codes, cd->ncodes * sizeof(pmix_status_t));
     index = pmix_globals.events.nhdlrs;
     multi->index = index;
+    multi->evhdlr = cd->evhdlr;
     ++pmix_globals.events.nhdlrs;
     multi->cbobject = cbobject;
     rc = _add_hdlr(&pmix_globals.events.multi_events, &multi->super,
@@ -396,9 +398,9 @@ static void reg_event_hdlr(int sd, short args, void *cbdata)
         PMIX_ERR_WOULD_BLOCK != rc) {
             /* unable to register */
         --pmix_globals.events.nhdlrs;
-    rc = PMIX_ERR_EVENT_REGISTRATION;
-    index = UINT_MAX;
-    goto ack;
+        rc = PMIX_ERR_EVENT_REGISTRATION;
+        index = UINT_MAX;
+        goto ack;
     }
     if (PMIX_ERR_WOULD_BLOCK == rc) {
         /* the callback will provide our response */