From 80031e260b3e8eb178e91257e7dffed25bd6e75a Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Fri, 10 Jul 2015 11:21:26 +0900 Subject: [PATCH 1/2] configury: fix XRC detection on OFED < 3.12 Thanks Paul Hargrove for the report (back-ported from commit open-mpi/ompi@9a49d42f81e3ec2b63aa4e7805d4c5e44a90abab) --- config/ompi_check_openfabrics.m4 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/ompi_check_openfabrics.m4 b/config/ompi_check_openfabrics.m4 index 797c9875fe..bb572ae8a1 100644 --- a/config/ompi_check_openfabrics.m4 +++ b/config/ompi_check_openfabrics.m4 @@ -168,7 +168,7 @@ AC_DEFUN([OMPI_CHECK_OPENFABRICS],[ # ibv_cmd_open_xrcd (aka XRC Domains) was added in OFED 3.12 if test "$enable_connectx_xrc" = "yes"; then $1_have_xrc=1 - AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp ibv_cmd_open_xrcd], + AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], [], [$1_have_xrc=0]) AC_CHECK_DECLS([IBV_SRQT_XRC], [], [$1_have_xrc=0], From bccbbb90edb5ed8a768046405a5432462592797b Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Fri, 10 Jul 2015 16:05:24 +0900 Subject: [PATCH 2/2] configury: fix (again) XRC detection on OFED < 3.12 since ibv_create_xrc_rcv_qp is now deprecated, and in order to be "future-proof", we have to consider the case in which only XRC Domains are supported. also, correctly handle distro that ship broken ibverbs devel headers. Thanks Paul Hargrove for the detailled report. (back-ported from commit open-mpi/ompi@f7cf7d50708e997be77ee320661790af7fe34f97) --- config/ompi_check_openfabrics.m4 | 59 +++++++++++++++++++++++--- ompi/mca/btl/openib/btl_openib.h | 4 +- ompi/mca/btl/openib/btl_openib_async.c | 11 ++--- ompi/mca/btl/openib/btl_openib_async.h | 2 +- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/config/ompi_check_openfabrics.m4 b/config/ompi_check_openfabrics.m4 index bb572ae8a1..192817e338 100644 --- a/config/ompi_check_openfabrics.m4 +++ b/config/ompi_check_openfabrics.m4 @@ -164,22 +164,67 @@ AC_DEFUN([OMPI_CHECK_OPENFABRICS],[ AC_CHECK_MEMBERS([struct ibv_device.transport_type], [], [], [#include ]) + # We have to check functions both exits *and* are declared + # since some distro ship broken ibverbs devel headers + # IBV_DEVICE_XRC is common to all OFED versions # ibv_create_xrc_rcv_qp was added in OFED 1.3 # ibv_cmd_open_xrcd (aka XRC Domains) was added in OFED 3.12 if test "$enable_connectx_xrc" = "yes"; then - $1_have_xrc=1 - AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], - [], [$1_have_xrc=0]) - AC_CHECK_DECLS([IBV_SRQT_XRC], - [], [$1_have_xrc=0], + AC_CHECK_DECLS([IBV_DEVICE_XRC], + [$1_have_xrc=1 + $1_have_xrc_domains=1], + [], [#include ]) fi if test "$enable_connectx_xrc" = "yes" \ && test $$1_have_xrc -eq 1; then - AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1]) + AC_CHECK_DECLS([ibv_create_xrc_rcv_qp], + [AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp], + [], [$1_have_xrc=0])], + [$1_have_xrc=0], + [#include ]) + fi + if test "$enable_connectx_xrc" = "yes" \ + && test $$1_have_xrc_domains -eq 1; then + AC_CHECK_DECLS([ibv_cmd_open_xrcd], + [AC_CHECK_DECLS([IBV_SRQT_XRC], + [AC_CHECK_FUNCS([ibv_cmd_open_xrcd], + [], [$1_have_xrc_domains=0])], + [$1_have_xrc_domains=0], + [#include ])], + [$1_have_xrc_domains=0], + [#include ]) + # XRC and XRC Domains should be considered as exclusive + if test "$$1_have_xrc" -eq 1 && \ + test "$$1_have_xrc_domains" -eq 1; then + $1_have_xrc=0 + fi fi - + if test "no" != "$enable_openib_dynamic_sl"; then + # We need ib_types.h file, which is installed with opensm-devel + # package. However, ib_types.h has a bad include directive, + # which will cause AC_CHECK_HEADER to fail. + # So instead, we will look for another file that is also + # installed as part of opensm-devel package and included in + # ib_types.h, but it doesn't include any other IB-related files. + AC_CHECK_HEADER([infiniband/complib/cl_types_osd.h], + [AC_CHECK_LIB([osmcomp], [cl_map_init], + [$1_have_opensm_devel=1],[])], + [], + []) + # Abort if dynamic SL support was explicitly requested but opensm-devel + # package wasn't found. Otherwise, OMPI will be built w/o dynamic SL. + AC_MSG_CHECKING([if can use dynamic SL support]) + AS_IF([test "$$1_have_opensm_devel" = "1"], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AS_IF([test "$enable_openib_dynamic_sl" = "yes"], + [AC_MSG_WARN([--enable-openib-dynamic-sl was specified but the]) + AC_MSG_WARN([appropriate header/library files could not be found]) + AC_MSG_WARN([Please install opensm-devel if you need dynamic SL support]) + AC_MSG_ERROR([Cannot continue])])]) + fi if test "no" != "$enable_openib_dynamic_sl"; then # We need ib_types.h file, which is installed with opensm-devel # package. However, ib_types.h has a bad include directive, diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index 98fbbd30d2..0a65c68422 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -55,8 +55,8 @@ BEGIN_C_DECLS -#define HAVE_XRC (1 == OMPI_HAVE_CONNECTX_XRC) -#define ENABLE_DYNAMIC_SL (1 == OMPI_ENABLE_DYNAMIC_SL) +#define HAVE_XRC (OMPI_HAVE_CONNECTX_XRC || OMPI_HAVE_CONNECTX_XRC_DOMAINS) +#define ENABLE_DYNAMIC_SL OMPI_ENABLE_DYNAMIC_SL #define MCA_BTL_IB_LEAVE_PINNED 1 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll diff --git a/ompi/mca/btl/openib/btl_openib_async.c b/ompi/mca/btl/openib/btl_openib_async.c index 0531b77c9b..b6a2cfb170 100644 --- a/ompi/mca/btl/openib/btl_openib_async.c +++ b/ompi/mca/btl/openib/btl_openib_async.c @@ -120,7 +120,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib return NULL; } -#if HAVE_XRC && !OMPI_HAVE_CONNECTX_XRC_DOMAINS +#if OMPI_HAVE_CONNECTX_XRC /* XRC recive QP to endpoint */ static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device) { @@ -350,16 +350,13 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po } event_type = event.event_type; -#if HAVE_XRC +#if OMPI_HAVE_CONNECTX_XRC /* is it XRC event ?*/ -#if OMPI_HAVE_CONNECTX_XRC_DOMAINS -#else if (IBV_XRC_QP_EVENT_FLAG & event.event_type) { xrc_event = true; /* Clean the bitnd handel as usual */ event_type ^= IBV_XRC_QP_EVENT_FLAG; } -#endif #endif switch(event_type) { case IBV_EVENT_PATH_MIG: @@ -369,7 +366,7 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po if (!xrc_event) mca_btl_openib_load_apm(event.element.qp, qp2endpoint(event.element.qp, device)); -#if HAVE_XRC && !OMPI_HAVE_CONNECTX_XRC_DOMAINS +#if OMPI_HAVE_CONNECTX_XRC else mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num, xrc_qp2endpoint(event.element.xrc_qp_num, device)); @@ -651,7 +648,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep) qp->qp_num, strerror(errno), errno)); } -#if HAVE_XRC && ! OMPI_HAVE_CONNECTX_XRC_DOMAINS +#if OMPI_HAVE_CONNECTX_XRC void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep) { struct ibv_qp_init_attr qp_init_attr; diff --git a/ompi/mca/btl/openib/btl_openib_async.h b/ompi/mca/btl/openib/btl_openib_async.h index 92a1042442..ea91dc870c 100644 --- a/ompi/mca/btl/openib/btl_openib_async.h +++ b/ompi/mca/btl/openib/btl_openib_async.h @@ -19,7 +19,7 @@ int start_async_event_thread(void); void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep); int btl_openib_async_command_done(int exp); -#if HAVE_XRC && ! OMPI_HAVE_CONNECTX_XRC_DOMAINS +#if OMPI_HAVE_CONNECTX_XRC void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep); #endif