openpmix · rhc54 · Feb 9, 2023 · Feb 2, 2023 · Feb 2, 2023 · Feb 4, 2023
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,4 +1,4 @@
-sphinx>=4.2.0,<=5.3.0
+sphinx>=4.2.0
 recommonmark
 docutils
-sphinx-rtd-theme<1.2.0
+sphinx-rtd-theme
diff --git a/examples/dmodex.c b/examples/dmodex.c
@@ -17,6 +17,7 @@
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * Copyright (c) 2019-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2021-2022 Nanook Consulting  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -38,7 +39,7 @@ static pmix_proc_t myproc;
 
 int main(int argc, char **argv)
 {
-    int rc;
+    int rc, np;
     pmix_value_t value;
     pmix_value_t *val = NULL;
     char tmp[1024];
@@ -57,7 +58,7 @@ int main(int argc, char **argv)
     if (PMIX_SUCCESS != (rc = PMIx_Init(&myproc, NULL, 0))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Init failed: %d\n", myproc.nspace, myproc.rank,
                 rc);
-        exit(0);
+        exit(EXIT_FAILURE);
     }
 
     /* get our job size */
@@ -144,7 +145,12 @@ int main(int argc, char **argv)
     }
     PMIX_ARGV_FREE(peers);
 
-    snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s",  myproc.nspace);
+    np = snprintf(proc.nspace, PMIX_MAX_NSLEN, "%s",  myproc.nspace);
+    if (np >= PMIX_MAX_NSLEN) {
+        fprintf(stderr, "Client ns %s rank %d: snprintf failed\n", myproc.nspace, myproc.rank);
+        exit(EXIT_FAILURE);
+    }
+
     PMIX_INFO_LOAD(&timeout, PMIX_TIMEOUT, &tlimit, PMIX_INT);
     /* get the committed data - ask for someone who doesn't exist as well */
     for (n = 0; n < nprocs; n++) {
@@ -236,7 +242,7 @@ int main(int argc, char **argv)
     if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
         fprintf(stderr, "Client ns %s rank %d: PMIx_Fence failed: %d\n", myproc.nspace, myproc.rank,
                 rc);
-        exit(1);
+        exit(EXIT_FAILURE);
     }
 
     if (PMIX_SUCCESS != (rc = PMIx_Finalize(NULL, 0))) {
@@ -249,5 +255,5 @@ int main(int argc, char **argv)
         }
     }
     fflush(stderr);
-    return (0);
+    return (EXIT_SUCCESS);
 }
diff --git a/examples/nodeid.c b/examples/nodeid.c
@@ -16,6 +16,7 @@
  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * Copyright (c) 2021-2022 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -77,18 +78,18 @@ int main(int argc, char **argv)
         proc.rank = n;
         rc = PMIx_Get(&proc, PMIX_NODEID, NULL, 0, &val);
         if (PMIX_SUCCESS != rc) {
-            fprintf(stderr, "[%s:%u] PMIx_Get failed for nodeid on rank %u: %s\n",
+            fprintf(stderr, "[%s:%u] PMIx_Get failed for nodeid on rank %zd: %s\n",
                     myproc.nspace, myproc.rank, n, PMIx_Error_string(rc));
             break;
         }
         PMIX_VALUE_GET_NUMBER(rc, val, nodeid, uint32_t);
         if (PMIX_SUCCESS != rc) {
-            fprintf(stderr, "[%s:%u] Got bad nodeid for rank %u: %s\n",
+            fprintf(stderr, "[%s:%u] Got bad nodeid for rank %zd: %s\n",
                     myproc.nspace, myproc.rank, n, PMIx_Error_string(rc));
             goto done;
         }
         if (0 == myproc.rank) {
-            fprintf(stderr, "[%s:%u] Peer %u is running on node %u\n",
+            fprintf(stderr, "[%s:%u] Peer %zd is running on node %u\n",
                     myproc.nspace, myproc.rank, n, nodeid);
         }
         PMIX_VALUE_RELEASE(val);

diff --git a/examples/pset.c b/examples/pset.c
@@ -16,6 +16,7 @@
  * Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.  All rights reserved.
  * Copyright (c) 2021-2022 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2023      Triad National Security, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -76,7 +77,7 @@ int main(int argc, char **argv)
                 myproc.nspace, myproc.rank, PMIx_Data_type_string(val->type));
         goto done;
     }
-    fprintf(stderr, "Client %s:%d PMIx_Get returned %d members\n", myproc.nspace, myproc.rank,
+    fprintf(stderr, "Client %s:%d PMIx_Get returned %zd members\n", myproc.nspace, myproc.rank,
             val->data.darray->size);
     pptr = (pmix_proc_t*)val->data.darray->array;
     for (n=0; n < val->data.darray->size; n++) {

diff --git a/src/hwloc/hwloc.c b/src/hwloc/hwloc.c
@@ -3,7 +3,7 @@
  * Copyright (c) 2013-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2021-2022 Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2021-2023 Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -334,21 +334,42 @@ int prte_hwloc_base_set_default_binding(void *jd, void *opt)
             pmix_output_verbose(options->verbosity, options->stream,
                                 "setdefaultbinding[%d] binding not given - using bypackage", __LINE__);
             PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_PACKAGE);
-        } else {
-            /* we are mapping by node or some other non-object method */
+        } else if (PRTE_MAPPING_PELIST == mpol) {
             if (options->use_hwthreads) {
                 /* if we are using hwthread cpus, then bind to those */
                 pmix_output_verbose(options->verbosity, options->stream,
-                                    "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__);
+                                    "setdefaultbinding[%d] binding not given - using byhwthread for pe-list", __LINE__);
                 PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding,
                                                 PRTE_BIND_TO_HWTHREAD);
             } else {
                 /* otherwise bind to core */
                 pmix_output_verbose(options->verbosity, options->stream,
-                                    "setdefaultbinding[%d] binding not given - using bycore", __LINE__);
+                                    "setdefaultbinding[%d] binding not given - using bycore for pe-list", __LINE__);
                 PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding,
                                                 PRTE_BIND_TO_CORE);
             }
+        } else {
+            if (options->nprocs <= 2) {
+                /* we are mapping by node or some other non-object method */
+                if (options->use_hwthreads) {
+                    /* if we are using hwthread cpus, then bind to those */
+                    pmix_output_verbose(options->verbosity, options->stream,
+                                        "setdefaultbinding[%d] binding not given - using byhwthread", __LINE__);
+                    PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding,
+                                                    PRTE_BIND_TO_HWTHREAD);
+                } else {
+                    /* otherwise bind to core */
+                    pmix_output_verbose(options->verbosity, options->stream,
+                                        "setdefaultbinding[%d] binding not given - using bycore", __LINE__);
+                    PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding,
+                                                    PRTE_BIND_TO_CORE);
+                }
+            } else {
+                pmix_output_verbose(options->verbosity, options->stream,
+                                    "setdefaultbinding[%d] binding not given - using bynuma",
+                                    __LINE__);
+                PRTE_SET_DEFAULT_BINDING_POLICY(jdata->map->binding, PRTE_BIND_TO_NUMA);
+            }
         }
     }
     /* they might have set the overload-allowed flag while wanting PRRTE

diff --git a/src/hwloc/hwloc_base_util.c b/src/hwloc/hwloc_base_util.c
@@ -169,11 +169,6 @@ hwloc_cpuset_t prte_hwloc_base_setup_summary(hwloc_topology_t topo)
     hwloc_cpuset_t avail = NULL;
 
     avail = hwloc_bitmap_alloc();
-    /* get the cpus we are bound to */
-    if (!prte_hwloc_synthetic_topo &&
-        0 <= hwloc_get_cpubind(topo, avail, HWLOC_CPUBIND_PROCESS)) {
-        return avail;
-    }
 
     /* get the root available cpuset */
 #if HWLOC_API_VERSION < 0x20000

diff --git a/src/mca/grpcomm/direct/grpcomm_direct.c b/src/mca/grpcomm/direct/grpcomm_direct.c
@@ -274,20 +274,6 @@ static void allgather_recv(int status, pmix_proc_t *sender,
             /* update the info with the collected value */
             info[n].value.type = PMIX_STATUS;
             info[n].value.data.status = coll->status;
-#ifdef PMIX_SIZE_ESTIMATE
-        } else if (PMIX_CHECK_KEY(&info[n], PMIX_SIZE_ESTIMATE)) {
-            PMIX_VALUE_GET_NUMBER(rc, &info[n].value, memsize, size_t);
-            if (PMIX_SUCCESS != rc) {
-                PMIX_ERROR_LOG(rc);
-                PMIX_PROC_FREE(sig.signature, sig.sz);
-                PMIX_DATA_BUFFER_DESTRUCT(&ctrlbuf);
-                return;
-            }
-            coll->memsize += memsize;
-            /* update the info with the collected value */
-            info[n].value.type = PMIX_SIZE;
-            info[n].value.data.size = coll->memsize;
-#endif
         } else if (PMIX_CHECK_KEY(&info[n], PMIX_GROUP_ASSIGN_CONTEXT_ID)) {
             assignID = PMIX_INFO_TRUE(&info[n]);
             if (assignID) {
@@ -346,18 +332,6 @@ static void allgather_recv(int status, pmix_proc_t *sender,
             }
             /* add some values to the payload in the bucket */
 
-#ifdef PMIX_SIZE_ESTIMATE
-            /* pack the memory size */
-            PMIX_INFO_LOAD(&infostat, PMIX_SIZE_ESTIMATE, &coll->memsize, PMIX_SIZE);
-            rc = PMIx_Data_pack(NULL, reply, &infostat, 1, PMIX_INFO);
-            PMIX_INFO_DESTRUCT(&infostat);
-            if (PMIX_SUCCESS != rc) {
-                PMIX_ERROR_LOG(rc);
-                PMIX_DATA_BUFFER_RELEASE(reply);
-                PMIX_PROC_FREE(sig.signature, sig.sz);
-                return;
-            }
-#endif
             /* if we were asked to provide a context id, do so */
             if (assignID) {
                 size_t sz;

diff --git a/src/mca/rmaps/base/help-prte-rmaps-base.txt b/src/mca/rmaps/base/help-prte-rmaps-base.txt
@@ -127,6 +127,36 @@ mapping operation.
   Mapping policy:      %s
   Binding policy:      %s
 
+#
+[span-packages-multiple]
+Your job failed to map because the resulting process placement
+would cause the process to be bound to CPUs in more than one
+package:
+
+  Mapping policy:  %s
+  Binding policy:  %s
+  CPUs/rank:       %d
+
+This configuration almost always results in a loss of performance
+that can significantly impact applications. Either alter the
+mapping, binding, and/or cpus/rank policies so that each process
+can fit into a single package, or consider using an alternative
+mapper that can handle this configuration (e.g., the rankfile mapper).
+#
+[span-packages-cpuset]
+Your job failed to map because the resulting process placement
+would cause the process to be bound to CPUs in more than one
+package:
+
+  Mapping policy:  %s
+  Binding policy:  %s
+  PE-LIST:         %s
+
+This configuration almost always results in a loss of performance
+that can significantly impact applications. Either alter the
+mapping, binding, and/or PE-LIST policies so that each process
+can fit into a single package, or consider using an alternative
+mapper that can handle this configuration (e.g., the rankfile mapper).
 #
 [unrecognized-policy]
 The specified %s directive is not recognized:

diff --git a/src/mca/rmaps/base/rmaps_base_binding.c b/src/mca/rmaps/base/rmaps_base_binding.c
@@ -181,9 +181,12 @@ static int bind_to_cpuset(prte_job_t *jdata,
     /* bind each process to prte_hwloc_base_cpu_list */
     unsigned idx;
     hwloc_bitmap_t tset;
-    hwloc_obj_t obj, root;
+    hwloc_obj_t obj, root, pkg;
     char **cpus;
     hwloc_obj_type_t type;
+    unsigned n, npkgs;
+    int rc;
+    bool included;
 
     pmix_output_verbose(5, prte_rmaps_base_framework.framework_output,
                         "mca:rmaps: bind job %s to cpus %s %s",
@@ -234,6 +237,31 @@ static int bind_to_cpuset(prte_job_t *jdata,
         /* bind the proc to all assigned cpus */
         tset = options->target;
     }
+    /* sanity check - are all the target cpus in a single
+     * package, or do they span packages?
+     */
+    npkgs = hwloc_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE);
+    included = false;
+    for (n=0; n < npkgs; n++) {
+        pkg = hwloc_get_obj_by_type(node->topology->topo, HWLOC_OBJ_PACKAGE, n);
+#if HWLOC_API_VERSION < 0x20000
+        rc = hwloc_bitmap_isincluded(tset, pkg->allowed_cpuset);
+#else
+        rc = hwloc_bitmap_isincluded(tset, pkg->cpuset);
+#endif
+        if (1 == rc) {
+            included = true;
+            break;
+        }
+    }
+    if (!included) {
+        pmix_show_help("help-prte-rmaps-base.txt", "span-packages-cpuset", true,
+                       prte_rmaps_base_print_mapping(jdata->map->mapping),
+                       prte_hwloc_base_print_binding(jdata->map->binding),
+                       options->cpuset);
+        PMIX_ARGV_FREE_COMPAT(cpus);
+        return PRTE_ERR_SILENT;
+    }
     /* bind to the specified cpuset */
     hwloc_bitmap_list_asprintf(&proc->cpuset, tset);
 
@@ -326,7 +354,11 @@ static int bind_multiple(prte_job_t *jdata, prte_proc_t *proc,
              * cover the request - so return an error */
             hwloc_bitmap_free(available);
             hwloc_bitmap_free(result);
-            return PRTE_ERR_TAKE_NEXT_OPTION;
+            pmix_show_help("help-prte-rmaps-base.txt", "span-packages-multiple", true,
+                           prte_rmaps_base_print_mapping(jdata->map->mapping),
+                           prte_hwloc_base_print_binding(jdata->map->binding),
+                           options->cpus_per_rank);
+            return PRTE_ERR_SILENT;
         }
     }
     /* we bind-to-cpu for the number of cpus that was specified,
@@ -406,7 +438,7 @@ int prte_rmaps_base_bind_proc(prte_job_t *jdata,
 
     if (1 < options->cpus_per_rank) {
         rc = bind_multiple(jdata, proc, node, obj, options);
-        if (PRTE_SUCCESS != rc && PRTE_ERR_TAKE_NEXT_OPTION != rc) {
+        if (PRTE_SUCCESS != rc) {
             PRTE_ERROR_LOG(rc);
         }
         return rc;

diff --git a/src/mca/rmaps/round_robin/rmaps_rr_mappers.c b/src/mca/rmaps/round_robin/rmaps_rr_mappers.c
@@ -144,6 +144,7 @@ int prte_rmaps_rr_byslot(prte_job_t *jdata,
             proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options);
             if (NULL == proc) {
                 /* move on to the next node */
+                rc = PRTE_ERR_SILENT;
                 break;
             }
             nprocs_mapped++;
@@ -305,6 +306,7 @@ int prte_rmaps_rr_bynode(prte_job_t *jdata,
             proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options);
             if (NULL == proc) {
                 /* move to next node */
+                rc = PRTE_ERR_SILENT;
                 break;
             }
             nprocs_mapped++;
@@ -466,7 +468,7 @@ int prte_rmaps_rr_bycpu(prte_job_t *jdata, prte_app_context_t *app,
         for (i = 0; i < options->nprocs && nprocs_mapped < app->num_procs; i++) {
             proc = prte_rmaps_base_setup_proc(jdata, app->idx, node, NULL, options);
             if (NULL == proc) {
-                rc = PRTE_ERR_OUT_OF_RESOURCE;
+                rc = PRTE_ERR_SILENT;
                 goto errout;
             }
             nprocs_mapped++;

diff --git a/src/mca/schizo/base/schizo_base_frame.c b/src/mca/schizo/base/schizo_base_frame.c
@@ -367,6 +367,8 @@ int prte_schizo_base_sanity(pmix_cli_result_t *cmd_line)
 
     char *outputs[] = {
         PRTE_CLI_TAG,
+        PRTE_CLI_TAG_DET,
+        PRTE_CLI_TAG_FULL,
         PRTE_CLI_RANK,
         PRTE_CLI_TIMESTAMP,
         PRTE_CLI_XML,

diff --git a/src/prted/pmix/pmix_server_dyn.c b/src/prted/pmix/pmix_server_dyn.c
@@ -981,9 +981,6 @@ static void connect_release(int status, pmix_data_buffer_t *buf, void *cbdata)
     uint32_t ctxid;
     bool first = true;
     char *payload;
-#ifdef PMIX_SIZE_ESTIMATE
-    size_t memsize = 0;
-#endif
 
     PMIX_ACQUIRE_OBJECT(md);
 
@@ -1002,15 +999,6 @@ static void connect_release(int status, pmix_data_buffer_t *buf, void *cbdata)
                     assignedID = true;
                     ++ninfo;
                 }
-#ifdef PMIX_SIZE_ESTIMATE
-            } else if (PMIX_CHECK_KEY(&infostat, PMIX_SIZE_ESTIMATE)) {
-                PMIX_VALUE_GET_NUMBER(rc, &infostat.value, memsize, size_t);
-                if (PMIX_SUCCESS != rc) {
-                    PMIX_ERROR_LOG(rc);
-                } else {
-                    ++ninfo;
-                }
-#endif
             }
             /* save where we are */
             payload = buf->unpack_ptr;
@@ -1033,11 +1021,6 @@ static void connect_release(int status, pmix_data_buffer_t *buf, void *cbdata)
             PMIX_INFO_LOAD(&info[n], PMIX_GROUP_CONTEXT_ID, &ctxid, PMIX_UINT32);
             ++n;
         }
-#ifdef PMIX_SIZE_ESTIMATE
-        if (0 < memsize) {
-            PMIX_INFO_LOAD(&info[n], PMIX_SIZE_ESTIMATE, &memsize, PMIX_SIZE);
-        }
-#endif
 
         /* there is a byte object for each proc in the connect operation */
         cnt = 1;