Skip to content

Topic/configury xrc 4 #718

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 13, 2015
Merged

Topic/configury xrc 4 #718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
66ef1c3
configury: fix fortran binding checks for ompi extensions
ggouaillardet Jul 13, 2015
f7cf7d5
configury: fix XRC detection on OFED < 3.12
ggouaillardet Jul 13, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions config/ompi_ext.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ EOF
# Make an AM conditional to see whether we're building the mpi_ext
# module. Note that we only build it if we support the ignore-tkr
# mpi module.
AS_IF([test $OMPI_BUILD_FORTRAN_USEMPI_BINDINGS -eq 1 && \
AS_IF([test $OMPI_BUILD_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPI_BINDINGS && \
test $OMPI_FORTRAN_HAVE_IGNORE_TKR -eq 1],
[OMPI_BUILD_FORTRAN_USEMPI_EXT=1],
[OMPI_BUILD_FORTRAN_USEMPI_EXT=0])
Expand Down Expand Up @@ -213,7 +213,7 @@ EOF

# Only build this mpi_f08_ext module if we're building the "use
# mpi_f08" module *and* it's the non-descriptor one.
AS_IF([test $OMPI_BUILD_FORTRAN_USEMPIF08_BINDINGS -eq 1 && \
AS_IF([test $OMPI_BUILD_FORTRAN_BINDINGS -ge $OMPI_FORTRAN_USEMPIF08_BINDINGS && \
test $OMPI_BUILD_FORTRAN_F08_SUBARRAYS -eq 0],
[OMPI_BUILD_FORTRAN_USEMPIF08_EXT=1],
[OMPI_BUILD_FORTRAN_USEMPIF08_EXT=0])
Expand Down
35 changes: 28 additions & 7 deletions config/opal_check_openfabrics.m4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,21 +148,42 @@ AC_DEFUN([OPAL_CHECK_OPENFABRICS],[
AC_CHECK_MEMBERS([struct ibv_device.transport_type], [], [],
[#include <infiniband/verbs.h>])

# We have to check functions both exits *and* are declared
# since some distro ship broken ibverbs devel headers
# IBV_DEVICE_XRC is common to all OFED versions
# ibv_create_xrc_rcv_qp was added in OFED 1.3
# ibv_cmd_open_xrcd (aka XRC Domains) was added in OFED 3.12
if test "$enable_connectx_xrc" = "yes"; then
$1_have_xrc=1
AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
[], [$1_have_xrc=0])
AC_CHECK_DECLS([IBV_SRQT_XRC],
[], [$1_have_xrc=0],
AC_CHECK_DECLS([IBV_DEVICE_XRC],
[$1_have_xrc=1
$1_have_xrc_domains=1],
[],
[#include <infiniband/verbs.h>])
fi
if test "$enable_connectx_xrc" = "yes" \
&& test $$1_have_xrc -eq 1; then
AC_CHECK_FUNCS([ibv_cmd_open_xrcd], [$1_have_xrc_domains=1])
AC_CHECK_DECLS([ibv_create_xrc_rcv_qp],
[AC_CHECK_FUNCS([ibv_create_xrc_rcv_qp],
[], [$1_have_xrc=0])],
[$1_have_xrc=0],
[#include <infiniband/driver.h>])
fi
if test "$enable_connectx_xrc" = "yes" \
&& test $$1_have_xrc_domains -eq 1; then
AC_CHECK_DECLS([ibv_cmd_open_xrcd],
[AC_CHECK_DECLS([IBV_SRQT_XRC],
[AC_CHECK_FUNCS([ibv_cmd_open_xrcd],
[], [$1_have_xrc_domains=0])],
[$1_have_xrc_domains=0],
[#include <infiniband/verbs.h>])],
[$1_have_xrc_domains=0],
[#include <infiniband/driver.h>])
# XRC and XRC Domains should be considered as exclusive
if test "$$1_have_xrc" -eq 1 && \
test "$$1_have_xrc_domains" -eq 1; then
$1_have_xrc=0
fi
fi


if test "no" != "$enable_openib_dynamic_sl"; then
# We need ib_types.h file, which is installed with opensm-devel
Expand Down
4 changes: 2 additions & 2 deletions opal/mca/btl/openib/btl_openib.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,8 @@

BEGIN_C_DECLS

#define HAVE_XRC (1 == OPAL_HAVE_CONNECTX_XRC)
#define ENABLE_DYNAMIC_SL (1 == OPAL_ENABLE_DYNAMIC_SL)
#define HAVE_XRC (OPAL_HAVE_CONNECTX_XRC || OPAL_HAVE_CONNECTX_XRC_DOMAINS)
#define ENABLE_DYNAMIC_SL OPAL_ENABLE_DYNAMIC_SL

#define MCA_BTL_IB_LEAVE_PINNED 1
#define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll
Expand Down
11 changes: 4 additions & 7 deletions opal/mca/btl/openib/btl_openib_async.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ static mca_btl_openib_endpoint_t * qp2endpoint(struct ibv_qp *qp, mca_btl_openib
return NULL;
}

#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
#if OPAL_HAVE_CONNECTX_XRC
/* XRC recive QP to endpoint */
static mca_btl_openib_endpoint_t * xrc_qp2endpoint(uint32_t qp_num, mca_btl_openib_device_t *device)
{
Expand Down Expand Up @@ -352,24 +352,21 @@ static int btl_openib_async_deviceh(struct mca_btl_openib_async_poll *devices_po
}

event_type = event.event_type;
#if HAVE_XRC
#if OPAL_HAVE_CONNECTX_XRC
/* is it XRC event ?*/
#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
#else
bool xrc_event = false;
if (IBV_XRC_QP_EVENT_FLAG & event.event_type) {
xrc_event = true;
/* Clean the bitnd handel as usual */
event_type ^= IBV_XRC_QP_EVENT_FLAG;
}
#endif
#endif
switch(event_type) {
case IBV_EVENT_PATH_MIG:
BTL_ERROR(("Alternative path migration event reported"));
if (APM_ENABLED) {
BTL_ERROR(("Trying to find additional path..."));
#if HAVE_XRC && !OPAL_HAVE_CONNECTX_XRC_DOMAINS
#if OPAL_HAVE_CONNECTX_XRC
if (xrc_event)
mca_btl_openib_load_apm_xrc_rcv(event.element.xrc_qp_num,
xrc_qp2endpoint(event.element.xrc_qp_num, device));
Expand Down Expand Up @@ -653,7 +650,7 @@ void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep)
qp->qp_num, strerror(errno), errno));
}

#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
#if OPAL_HAVE_CONNECTX_XRC
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep)
{
struct ibv_qp_init_attr qp_init_attr;
Expand Down
2 changes: 1 addition & 1 deletion opal/mca/btl/openib/btl_openib_async.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
int start_async_event_thread(void);
void mca_btl_openib_load_apm(struct ibv_qp *qp, mca_btl_openib_endpoint_t *ep);
int btl_openib_async_command_done(int exp);
#if HAVE_XRC && ! OPAL_HAVE_CONNECTX_XRC_DOMAINS
#if OPAL_HAVE_CONNECTX_XRC
void mca_btl_openib_load_apm_xrc_rcv(uint32_t qp_num, mca_btl_openib_endpoint_t *ep);
#endif

Expand Down
7 changes: 6 additions & 1 deletion opal/mca/btl/openib/btl_openib_atomic.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,12 @@ static int mca_btl_openib_atomic_internal (struct mca_btl_base_module_t *btl, st

#if HAVE_XRC
if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
frag->sr_desc.xrc_remote_srq_num=endpoint->rem_info.rem_srqs[qp].rem_srq_num;
#if OPAL_HAVE_CONNECTX_XRC_DOMAINS
frag->sr_desc.qp_type.xrc.remote_srqn = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
#else
frag->sr_desc.xrc_remote_srq_num = endpoint->rem_info.rem_srqs[qp].rem_srq_num;
#endif

}
#endif

Expand Down
7 changes: 5 additions & 2 deletions opal/mca/btl/openib/btl_openib_put.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,13 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint

#if HAVE_XRC
if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) {
#if OPAL_HAVE_CONNECTX_XRC_DOMAINS

#if OPAL_HAVE_CONNECTX_XRC
to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
#elif OPAL_HAVE_CONNECTX_XRC_DOMAINS
to_out_frag(frag)->sr_desc.qp_type.xrc.remote_srqn = ep->rem_info.rem_srqs[qp].rem_srq_num;
#else
to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num;
#error "that should never happen"
#endif
}
#endif
Expand Down