From 5e13e1ab7d2b30eb70a98907c9063657df12b415 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 26 Jul 2016 16:01:11 -0600 Subject: [PATCH] btl/openib: set send flags only after endpoint is connected The max inline send size on a queue pair is not available until after the endpoint is connected. Before this commit the send flags (including the inline flag) were set before this value was initialized. This commit moves setting the send_flags down to mca_btl_openib_put_internal which is only called after the endpoint is connected. This fixes a bug when using osc/rdma. Signed-off-by: Nathan Hjelm --- opal/mca/btl/openib/btl_openib_put.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/opal/mca/btl/openib/btl_openib_put.c b/opal/mca/btl/openib/btl_openib_put.c index 2a9ee2ddd67..83260e05446 100644 --- a/opal/mca/btl/openib/btl_openib_put.c +++ b/opal/mca/btl/openib/btl_openib_put.c @@ -85,7 +85,6 @@ int mca_btl_openib_put (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint /* post descriptor */ to_out_frag(frag)->sr_desc.opcode = IBV_WR_RDMA_WRITE; - to_out_frag(frag)->sr_desc.send_flags = ib_send_flags(size, &(ep->qps[qp]), 1); to_out_frag(frag)->sr_desc.wr.rdma.remote_addr = remote_address; qp_inflight_wqe_to_frag(ep, qp, to_com_frag(frag)); @@ -140,11 +139,14 @@ int mca_btl_openib_put_internal (mca_btl_base_module_t *btl, struct mca_btl_base struct ibv_send_wr *bad_wr; int rc; + /* NTH: the inline send size and remote SRQ number are only available once the endpoint is + * connected. By setting these values here instead of mca_btl_openib_put we guarantee + * both fields are initialized */ + to_out_frag(frag)->sr_desc.send_flags = ib_send_flags (to_com_frag(frag)->sg_entry.length, + &(ep->qps[qp]), 1); + #if HAVE_XRC if (MCA_BTL_XRC_ENABLED && BTL_OPENIB_QP_TYPE_XRC(qp)) { - /* NTH: the remote SRQ number is only available once the endpoint is connected. By - * setting the value here instead of mca_btl_openib_put we guarantee the rem_srqs - * array is initialized. */ #if OPAL_HAVE_CONNECTX_XRC to_out_frag(frag)->sr_desc.xrc_remote_srq_num = ep->rem_info.rem_srqs[qp].rem_srq_num; #elif OPAL_HAVE_CONNECTX_XRC_DOMAINS