diff --git a/oshmem/mca/sshmem/verbs/configure.m4 b/oshmem/mca/sshmem/verbs/configure.m4 index cf67bc4dfb..fbb4cfa34d 100644 --- a/oshmem/mca/sshmem/verbs/configure.m4 +++ b/oshmem/mca/sshmem/verbs/configure.m4 @@ -72,6 +72,8 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[ oshmem_verbs_sm_build_verbs=0 fi ]) + + exp_reg_mr_happy=0 AS_IF([test "$oshmem_have_mpage" = "3"], [ AC_CHECK_MEMBER([struct ibv_exp_reg_shared_mr_in.exp_access], @@ -80,7 +82,13 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[ )], [], [#include ]) + + AC_CHECK_MEMBER([struct ibv_exp_reg_mr_in.create_flags], + [exp_reg_mr_happy=1], + [], + [#include ]) ]) + AC_DEFINE_UNQUOTED(MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS, $exp_reg_mr_happy, [create_flags field is part of ibv_exp_reg_mr_in]), AS_IF([test "$enable_verbs_sshmem" = "yes" -a "$oshmem_verbs_sm_build_verbs" = "0"], [AC_MSG_WARN([VERBS shared memory support requested but not found]) diff --git a/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c b/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c index 420cad3c40..483fe7faa4 100644 --- a/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c +++ b/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c @@ -188,6 +188,9 @@ verbs_runtime_query(mca_base_module_t **module, mca_sshmem_verbs_fill_shared_mr(&in_smr, device->ib_pd, device->ib_mr_shared->handle, addr, access_flag); ib_mr = ibv_exp_reg_shared_mr(&in_smr); if (NULL == ib_mr) { + if (mca_sshmem_verbs_component.has_shared_mr == 1) + rc = OSHMEM_ERR_OUT_OF_RESOURCE; + mca_sshmem_verbs_component.has_shared_mr = 0; rc = OSHMEM_ERR_OUT_OF_RESOURCE; } else { @@ -198,6 +201,14 @@ verbs_runtime_query(mca_base_module_t **module, #endif /* MPAGE_ENABLE */ } +#if !MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS + /* disqualify ourselves if we can not alloc contig + * pages at fixed address + */ + if (mca_sshmem_verbs_component.has_shared_mr == 0) + rc = OSHMEM_ERR_OUT_OF_RESOURCE; +#endif + /* all is well - rainbows and butterflies */ if (!rc) { *priority = mca_sshmem_verbs_component.priority; diff --git a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c index efd6b55124..4b699da943 100644 --- a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c +++ b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c @@ -260,6 +260,15 @@ segment_create(map_segment_t *ds_buf, #endif /* MPAGE_ENABLE */ struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0}; + +#if MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS + if (0 == mca_sshmem_verbs_component.has_shared_mr) { + in.addr = (void *)mca_sshmem_base_start_address; + in.comp_mask = IBV_EXP_REG_MR_CREATE_FLAGS; + in.create_flags = IBV_EXP_REG_MR_CREATE_CONTIG; + in.exp_access = access_flag; + } +#endif ib_mr = ibv_exp_reg_mr(&in); if (NULL == ib_mr) { OPAL_OUTPUT_VERBOSE(