From 984d204cc2355388868b3486062dd6044a212cc3 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Sat, 6 Feb 2016 06:00:00 -0800 Subject: [PATCH 1/4] usnic: ensure to init module->av_eq_num (cherry picked from open-mpi/ompi@5124039) --- ompi/mca/btl/usnic/btl_usnic_module.c | 1 + 1 file changed, 1 insertion(+) diff --git a/ompi/mca/btl/usnic/btl_usnic_module.c b/ompi/mca/btl/usnic/btl_usnic_module.c index 799282bdbd..cbf14a939c 100644 --- a/ompi/mca/btl/usnic/btl_usnic_module.c +++ b/ompi/mca/btl/usnic/btl_usnic_module.c @@ -1895,6 +1895,7 @@ static void init_queue_lengths(opal_btl_usnic_module_t *module) } else { module->cq_num = mca_btl_usnic_component.cq_num; } + module->av_eq_num = mca_btl_usnic_component.av_eq_num; /* * Queue sizes for priority channel scale with # of endpoint. A From 2e1e9ee0db0af7860ff6060284fe1487032ba97f Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Fri, 5 Feb 2016 12:22:39 -0800 Subject: [PATCH 2/4] usnic: ensure to use ntohl() for network-order values (cherry-picked from open-mpi/ompi@dac2fe1) --- ompi/mca/btl/usnic/btl_usnic_endpoint.h | 1 + ompi/mca/btl/usnic/btl_usnic_util.c | 17 ++++++++++------- ompi/mca/btl/usnic/btl_usnic_util.h | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ompi/mca/btl/usnic/btl_usnic_endpoint.h b/ompi/mca/btl/usnic/btl_usnic_endpoint.h index c76eee6d95..f428a28e64 100644 --- a/ompi/mca/btl/usnic/btl_usnic_endpoint.h +++ b/ompi/mca/btl/usnic/btl_usnic_endpoint.h @@ -69,6 +69,7 @@ typedef struct opal_btl_usnic_modex_t { uint32_t ipv4_addr; /* Stored in host order */ uint32_t ports[USNIC_NUM_CHANNELS]; + /* Stored in network order */ uint32_t netmask; /* Stored in host order */ uint32_t connectivity_udp_port; diff --git a/ompi/mca/btl/usnic/btl_usnic_util.c b/ompi/mca/btl/usnic/btl_usnic_util.c index d98a790a0a..4ec1ecbe67 100644 --- a/ompi/mca/btl/usnic/btl_usnic_util.c +++ b/ompi/mca/btl/usnic/btl_usnic_util.c @@ -115,24 +115,27 @@ opal_btl_usnic_dump_hex(void *vaddr, int len) * using inet_ntop()). */ void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen, - uint32_t addr, uint32_t netmask) + uint32_t addr_be, uint32_t netmask_be) { int prefixlen; + uint32_t netmask = ntohl(netmask_be); + uint32_t addr = ntohl(addr_be); uint8_t *p = (uint8_t*) &addr; + if (netmask != 0) { prefixlen = 33 - ffs(netmask); snprintf(out, maxlen, "%u.%u.%u.%u/%u", - p[0], - p[1], - p[2], p[3], + p[2], + p[1], + p[0], prefixlen); } else { snprintf(out, maxlen, "%u.%u.%u.%u", - p[0], - p[1], + p[3], p[2], - p[3]); + p[1], + p[0]); } } diff --git a/ompi/mca/btl/usnic/btl_usnic_util.h b/ompi/mca/btl/usnic/btl_usnic_util.h index 4fd08f93b9..54714fb2f7 100644 --- a/ompi/mca/btl/usnic/btl_usnic_util.h +++ b/ompi/mca/btl/usnic/btl_usnic_util.h @@ -113,7 +113,7 @@ void opal_btl_usnic_util_abort(const char *msg, const char *file, int line); * expected to be in network byte order. */ void opal_btl_usnic_snprintf_ipv4_addr(char *out, size_t maxlen, - uint32_t addr, uint32_t netmask); + uint32_t addr_be, uint32_t netmask_be); void opal_btl_usnic_snprintf_bool_array(char *s, size_t slen, bool a[], size_t alen); From cc6d9abbb11aa76ce597ba8c195e128d29627509 Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Fri, 5 Feb 2016 12:27:30 -0800 Subject: [PATCH 3/4] usnic: add high-verbose reachability messages (cherry-picked from open-mpi/ompi@87dbe6c) --- ompi/mca/btl/usnic/btl_usnic_module.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ompi/mca/btl/usnic/btl_usnic_module.c b/ompi/mca/btl/usnic/btl_usnic_module.c index cbf14a939c..53c5640618 100644 --- a/ompi/mca/btl/usnic/btl_usnic_module.c +++ b/ompi/mca/btl/usnic/btl_usnic_module.c @@ -98,11 +98,18 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module, /* Do not create loopback usnic connections */ if (opal_proc == my_proc) { + opal_output_verbose(75, USNIC_OUT, + "btl:usnic:add_procs:%s: not connecting to self", + module->fabric_info->fabric_attr->name); continue; } /* usNIC does not support loopback to the same machine */ if (OPAL_PROC_ON_LOCAL_NODE(opal_proc->proc_flags)) { + opal_output_verbose(75, USNIC_OUT, + "btl:usnic:add_procs:%s: not connecting to %s on same server", + module->fabric_info->fabric_attr->name, + usnic_compat_proc_name_print(&opal_proc->proc_name)); continue; } @@ -115,6 +122,11 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module, if (OPAL_ERR_UNREACH == rc) { /* If the peer doesn't have usnic modex info, then we just skip it */ + opal_output_verbose(75, USNIC_OUT, + "btl:usnic:add_procs:%s: peer %s on %s does not have usnic modex info; skipping", + module->fabric_info->fabric_attr->name, + usnic_compat_proc_name_print(&opal_proc->proc_name), + opal_get_proc_hostname(opal_proc)); continue; } else if (OPAL_SUCCESS != rc) { return OPAL_ERR_OUT_OF_RESOURCE; From 564aede3f5020c01dcc4a27a55f9152d6bafb13e Mon Sep 17 00:00:00 2001 From: Jeff Squyres Date: Fri, 5 Feb 2016 12:27:57 -0800 Subject: [PATCH 4/4] usnic: update a few verbose reachability messages (cherry-picked from open-mpi/ompi@8d0a592) --- ompi/mca/btl/usnic/btl_usnic_module.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ompi/mca/btl/usnic/btl_usnic_module.c b/ompi/mca/btl/usnic/btl_usnic_module.c index 53c5640618..53696ce0fa 100644 --- a/ompi/mca/btl/usnic/btl_usnic_module.c +++ b/ompi/mca/btl/usnic/btl_usnic_module.c @@ -139,8 +139,10 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module, &usnic_endpoint); if (OPAL_SUCCESS != rc) { opal_output_verbose(5, USNIC_OUT, - "btl:usnic:%s: unable to create endpoint for module=%p proc=%p\n", - __func__, (void *)module, (void *)usnic_proc); + "btl:usnic:add_procs:%s: unable to create endpoint to peer %s on %s", + module->fabric_info->fabric_attr->name, + usnic_compat_proc_name_print(&opal_proc->proc_name), + opal_get_proc_hostname(opal_proc)); OBJ_RELEASE(usnic_proc); continue; } @@ -156,7 +158,8 @@ static int add_procs_block_create_endpoints(opal_btl_usnic_module_t *module, modex->netmask); opal_output_verbose(5, USNIC_OUT, - "btl:usnic: new usnic peer endpoint: %s, proirity port %d, data port %d", + "btl:usnic:add_procs:%s: new usnic peer endpoint: %s, proirity port %d, data port %d", + module->fabric_info->fabric_attr->name, str, modex->ports[USNIC_PRIORITY_CHANNEL], modex->ports[USNIC_DATA_CHANNEL]);