From eaba98ce5d6f31f18437d1e4e36b7d73e422e854 Mon Sep 17 00:00:00 2001 From: Howard Pritchard Date: Tue, 13 Oct 2015 10:25:13 -0500 Subject: [PATCH] btl/ugni: fix very poor aries bw problem The handling of RDMA get alignment in ugni BTL for Aries (cray xc) was wrong, resulting in very poor bandwidth for ugni BTL on aries. Verified using osu_bw now gives sensible bandwidth on Aries. Fixes #1005 Signed-off-by: Howard Pritchard --- opal/mca/btl/ugni/btl_ugni_component.c | 3 +++ opal/mca/btl/ugni/btl_ugni_get.c | 9 ++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/opal/mca/btl/ugni/btl_ugni_component.c b/opal/mca/btl/ugni/btl_ugni_component.c index f687e72d26e..409a10e803e 100644 --- a/opal/mca/btl/ugni/btl_ugni_component.c +++ b/opal/mca/btl/ugni/btl_ugni_component.c @@ -231,6 +231,9 @@ btl_ugni_component_register(void) /* determine if there are get alignment restrictions */ GNI_GetDeviceType (&device_type); + /* + * see def. of ALIGNMENT_MASK to figure this one out + */ if (GNI_DEVICE_GEMINI == device_type) { mca_btl_ugni_module.super.btl_get_alignment = 4; } else { diff --git a/opal/mca/btl/ugni/btl_ugni_get.c b/opal/mca/btl/ugni/btl_ugni_get.c index f244035a56c..e152939d96a 100644 --- a/opal/mca/btl/ugni/btl_ugni_get.c +++ b/opal/mca/btl/ugni/btl_ugni_get.c @@ -13,6 +13,12 @@ #include "btl_ugni_rdma.h" #include "btl_ugni_smsg.h" +/* + * taken from osc_rdma_comm.h, ugh. + */ + +#define ALIGNMENT_MASK(x) ((x) ? (x) - 1 : 0) + int mca_btl_ugni_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle, mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags, @@ -21,7 +27,8 @@ int mca_btl_ugni_get (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t bool check; /* Check if the get is aligned/sized on a multiple of 4 */ - check = !!((remote_address | (uint64_t)(intptr_t) local_address | size) & (mca_btl_ugni_module.super.btl_get_alignment - 1)); + check = !!((remote_address | (uint64_t)(intptr_t) local_address | size) & + ALIGNMENT_MASK(mca_btl_ugni_module.super.btl_get_alignment)); if (OPAL_UNLIKELY(check || size > mca_btl_ugni_module.super.btl_get_limit)) { BTL_VERBOSE(("RDMA/FMA Get not available due to size or alignment restrictions"));