From 4dc487fc3d6353022a68717f17a6cda8fa15877d Mon Sep 17 00:00:00 2001 From: Mike Dubman Date: Mon, 22 Sep 2014 20:11:40 +0300 Subject: [PATCH] OSHMEM: sshmem verbs: allocate memory at fixed address Use experimental verbs to allocate memory at fixed base virtual address. verbs will disqualify itself if shared_mr is disabled or not supported and it is impossible to allocate memory starting at fixed base virtual address. verbs contig pages allocator did not guarantee fixed va, now it does. (cherry picked from commit fd77ebd4525e9e0c1a3ab1c4966bf31aa45251b4) Apply Jeff`s comments Update with Jeff commits --- oshmem/mca/sshmem/verbs/configure.m4 | 8 ++++++++ oshmem/mca/sshmem/verbs/sshmem_verbs_component.c | 11 +++++++++++ oshmem/mca/sshmem/verbs/sshmem_verbs_module.c | 9 +++++++++ 3 files changed, 28 insertions(+) diff --git a/oshmem/mca/sshmem/verbs/configure.m4 b/oshmem/mca/sshmem/verbs/configure.m4 index cf67bc4dfb..fbb4cfa34d 100644 --- a/oshmem/mca/sshmem/verbs/configure.m4 +++ b/oshmem/mca/sshmem/verbs/configure.m4 @@ -72,6 +72,8 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[ oshmem_verbs_sm_build_verbs=0 fi ]) + + exp_reg_mr_happy=0 AS_IF([test "$oshmem_have_mpage" = "3"], [ AC_CHECK_MEMBER([struct ibv_exp_reg_shared_mr_in.exp_access], @@ -80,7 +82,13 @@ AC_DEFUN([MCA_oshmem_sshmem_verbs_CONFIG],[ )], [], [#include ]) + + AC_CHECK_MEMBER([struct ibv_exp_reg_mr_in.create_flags], + [exp_reg_mr_happy=1], + [], + [#include ]) ]) + AC_DEFINE_UNQUOTED(MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS, $exp_reg_mr_happy, [create_flags field is part of ibv_exp_reg_mr_in]), AS_IF([test "$enable_verbs_sshmem" = "yes" -a "$oshmem_verbs_sm_build_verbs" = "0"], [AC_MSG_WARN([VERBS shared memory support requested but not found]) diff --git a/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c b/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c index 420cad3c40..483fe7faa4 100644 --- a/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c +++ b/oshmem/mca/sshmem/verbs/sshmem_verbs_component.c @@ -188,6 +188,9 @@ verbs_runtime_query(mca_base_module_t **module, mca_sshmem_verbs_fill_shared_mr(&in_smr, device->ib_pd, device->ib_mr_shared->handle, addr, access_flag); ib_mr = ibv_exp_reg_shared_mr(&in_smr); if (NULL == ib_mr) { + if (mca_sshmem_verbs_component.has_shared_mr == 1) + rc = OSHMEM_ERR_OUT_OF_RESOURCE; + mca_sshmem_verbs_component.has_shared_mr = 0; rc = OSHMEM_ERR_OUT_OF_RESOURCE; } else { @@ -198,6 +201,14 @@ verbs_runtime_query(mca_base_module_t **module, #endif /* MPAGE_ENABLE */ } +#if !MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS + /* disqualify ourselves if we can not alloc contig + * pages at fixed address + */ + if (mca_sshmem_verbs_component.has_shared_mr == 0) + rc = OSHMEM_ERR_OUT_OF_RESOURCE; +#endif + /* all is well - rainbows and butterflies */ if (!rc) { *priority = mca_sshmem_verbs_component.priority; diff --git a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c index efd6b55124..4b699da943 100644 --- a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c +++ b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c @@ -260,6 +260,15 @@ segment_create(map_segment_t *ds_buf, #endif /* MPAGE_ENABLE */ struct ibv_exp_reg_mr_in in = {device->ib_pd, addr, size, access_flag|exp_access_flag, 0}; + +#if MPAGE_HAVE_IBV_EXP_REG_MR_CREATE_FLAGS + if (0 == mca_sshmem_verbs_component.has_shared_mr) { + in.addr = (void *)mca_sshmem_base_start_address; + in.comp_mask = IBV_EXP_REG_MR_CREATE_FLAGS; + in.create_flags = IBV_EXP_REG_MR_CREATE_CONTIG; + in.exp_access = access_flag; + } +#endif ib_mr = ibv_exp_reg_mr(&in); if (NULL == ib_mr) { OPAL_OUTPUT_VERBOSE(