open-mpi · hjelmn · Mar 11, 2015 · Mar 10, 2015
diff --git a/opal/mca/btl/openib/btl_openib.c b/opal/mca/btl/openib/btl_openib.c
@@ -1130,7 +1130,7 @@ ib_frag_alloc(mca_btl_openib_module_t *btl, size_t size, uint8_t order,
         return NULL;
 
     /* not all upper layer users set this */
-    to_base_frag(item)->segment.base.seg_len = size;
+    to_base_frag(item)->segment.seg_len = size;
     to_base_frag(item)->base.order = order;
     to_base_frag(item)->base.des_flags = flags;
 
@@ -1161,7 +1161,7 @@ static mca_btl_openib_send_frag_t *check_coalescing(opal_list_t *frag_list,
         }
 
         total_length = size + frag->coalesced_length +
-            to_base_frag(frag)->segment.base.seg_len +
+            to_base_frag(frag)->segment.seg_len +
             sizeof(mca_btl_openib_header_coalesced_t);
 
         qp = to_base_frag(frag)->base.order;
@@ -1248,8 +1248,8 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc(
         sfrag->hdr->tag = MCA_BTL_TAG_IB;
         ctrl_hdr->type = MCA_BTL_OPENIB_CONTROL_COALESCED;
         clsc_hdr->tag = org_tag;
-        clsc_hdr->size = to_base_frag(sfrag)->segment.base.seg_len;
-        clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.base.seg_len;
+        clsc_hdr->size = to_base_frag(sfrag)->segment.seg_len;
+        clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.seg_len;
         if(ep->nbo)
             BTL_OPENIB_HEADER_COALESCED_HTON(*clsc_hdr);
         sfrag->coalesced_length = sizeof(mca_btl_openib_control_header_t) +
@@ -1259,13 +1259,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc(
 
     cfrag->hdr = (mca_btl_openib_header_coalesced_t*)((unsigned char*)(sfrag->hdr + 1) + 
                   sfrag->coalesced_length +
-                  to_base_frag(sfrag)->segment.base.seg_len);
+                  to_base_frag(sfrag)->segment.seg_len);
     cfrag->hdr = (mca_btl_openib_header_coalesced_t*)BTL_OPENIB_ALIGN_COALESCE_HDR(cfrag->hdr);
     cfrag->hdr->alloc_size = size;
 
     /* point coalesced frag pointer into a data buffer */
-    to_base_frag(cfrag)->segment.base.seg_addr.pval = cfrag->hdr + 1;
-    to_base_frag(cfrag)->segment.base.seg_len = size;
+    to_base_frag(cfrag)->segment.seg_addr.pval = cfrag->hdr + 1;
+    to_base_frag(cfrag)->segment.seg_len = size;
 
     /* NTH: there is no reason to append the coalesced fragment here. No more
      * fragments will be added until either send or free has been called on
@@ -1297,7 +1297,7 @@ int mca_btl_openib_free(
             to_com_frag(des)->sg_entry.addr =
                 (uint64_t)(uintptr_t)to_send_frag(des)->hdr;
             to_send_frag(des)->coalesced_length = 0;
-            to_base_frag(des)->segment.base.seg_addr.pval =
+            to_base_frag(des)->segment.seg_addr.pval =
                 to_send_frag(des)->hdr + 1;
             assert(!opal_list_get_size(&to_send_frag(des)->coalesced_frags));
             /* fall through */
@@ -1369,7 +1369,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
         return NULL;
     }
 
-    ptr = to_base_frag(frag)->segment.base.seg_addr.pval;
+    ptr = to_base_frag(frag)->segment.seg_addr.pval;
 
     iov.iov_len = max_data;
     iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*) ptr + reserve );
@@ -1388,7 +1388,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
     *size = max_data;
 
     /* not all upper layer users set this */
-    to_base_frag(frag)->segment.base.seg_len = max_data + reserve;
+    to_base_frag(frag)->segment.seg_len = max_data + reserve;
 
     return &to_base_frag(frag)->base;
 }
@@ -1593,22 +1593,22 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl,
         goto cant_send_frag;
     }
 
-    frag->segment.base.seg_len = size;
+    frag->segment.seg_len = size;
     frag->base.order = qp;
     frag->base.des_flags = flags;
     hdr->tag = tag;
     to_com_frag(item)->endpoint = ep;
 
     /* put match header */
-    memcpy(frag->segment.base.seg_addr.pval, header, header_size);
+    memcpy(frag->segment.seg_addr.pval, header, header_size);
 
     /* Pack data */
     if(payload_size) {
         size_t max_data;
         struct iovec iov;
         uint32_t iov_count;
         /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size);
+        iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
         iov.iov_len  = max_data = payload_size;
         iov_count    = 1;
 

diff --git a/opal/mca/btl/openib/btl_openib_component.c b/opal/mca/btl/openib/btl_openib_component.c
@@ -463,7 +463,7 @@ static void btl_openib_control(mca_btl_base_module_t* btl,
     mca_btl_openib_module_t *obtl = (mca_btl_openib_module_t*)btl;
     mca_btl_openib_endpoint_t* ep = to_com_frag(des)->endpoint;
     mca_btl_openib_control_header_t *ctl_hdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.base.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.seg_addr.pval;
     mca_btl_openib_eager_rdma_header_t *rdma_hdr;
     mca_btl_openib_header_coalesced_t *clsc_hdr =
         (mca_btl_openib_header_coalesced_t*)(ctl_hdr + 1);
@@ -1179,15 +1179,15 @@ static void merge_values(opal_btl_openib_ini_values_t *target,
 static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
 {
     mca_btl_openib_control_header_t* chdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
     return (MCA_BTL_TAG_IB == frag->hdr->tag) &&
         (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type);
 }
 
 static bool inline is_cts_message(const mca_btl_openib_recv_frag_t *frag)
 {
     mca_btl_openib_control_header_t* chdr =
-        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
+        (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
     return (MCA_BTL_TAG_IB == frag->hdr->tag) &&
         (MCA_BTL_OPENIB_CONTROL_CTS == chdr->type);
 }
@@ -3703,7 +3703,7 @@ static int progress_one_device(mca_btl_openib_device_t *device)
             OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
             frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
                 size - BTL_OPENIB_FTR_PADDING(size) + sizeof(mca_btl_openib_footer_t));
-            to_base_frag(frag)->segment.base.seg_addr.pval =
+            to_base_frag(frag)->segment.seg_addr.pval =
                 ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
 
             ret = btl_openib_handle_incoming(btl, to_com_frag(frag)->endpoint,

diff --git a/opal/mca/btl/openib/btl_openib_endpoint.c b/opal/mca/btl/openib/btl_openib_endpoint.c
@@ -492,15 +492,15 @@ void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint)
     base_des->des_cbdata = NULL;
     base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     base_des->order = mca_btl_openib_component.credits_qp;
-    openib_frag->segment.base.seg_len = sizeof(mca_btl_openib_control_header_t);
+    openib_frag->segment.seg_len = sizeof(mca_btl_openib_control_header_t);
     com_frag->endpoint = endpoint;
 
     sc_frag->hdr->tag = MCA_BTL_TAG_IB;
     sc_frag->hdr->cm_seen = 0;
     sc_frag->hdr->credits = 0;
 
     ctl_hdr = (mca_btl_openib_control_header_t*)
-        openib_frag->segment.base.seg_addr.pval;
+        openib_frag->segment.seg_addr.pval;
     ctl_hdr->type = MCA_BTL_OPENIB_CONTROL_CTS;
 
     /* Send the fragment */
@@ -727,13 +727,13 @@ void mca_btl_openib_endpoint_send_credits(mca_btl_openib_endpoint_t* endpoint,
         to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;;
         to_com_frag(frag)->endpoint = endpoint;
         frag->hdr->tag = MCA_BTL_TAG_IB;
-        to_base_frag(frag)->segment.base.seg_len =
+        to_base_frag(frag)->segment.seg_len =
             sizeof(mca_btl_openib_rdma_credits_header_t);
     }
 
     assert(frag->qp_idx == qp);
     credits_hdr = (mca_btl_openib_rdma_credits_header_t*)
-        to_base_frag(frag)->segment.base.seg_addr.pval;
+        to_base_frag(frag)->segment.seg_addr.pval;
     if(OPAL_SUCCESS == acquire_eager_rdma_send_credit(endpoint)) {
         do_rdma = true;
     } else {
@@ -818,12 +818,12 @@ static int mca_btl_openib_endpoint_send_eager_rdma(
     to_base_frag(frag)->base.des_cbdata = NULL;
     to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
-    to_base_frag(frag)->segment.base.seg_len =
+    to_base_frag(frag)->segment.seg_len =
         sizeof(mca_btl_openib_eager_rdma_header_t);
     to_com_frag(frag)->endpoint = endpoint;
 
     frag->hdr->tag = MCA_BTL_TAG_IB;
-    rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
+    rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
     rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA;
     rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey;
     rdma_hdr->rdma_start.lval = opal_ptr_ptol(endpoint->eager_rdma_local.base.pval);
@@ -925,7 +925,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
         to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA;
         to_com_frag(frag)->endpoint = endpoint;
         frag->ftr = (mca_btl_openib_footer_t*)
-            ((char*)to_base_frag(frag)->segment.base.seg_addr.pval +
+            ((char*)to_base_frag(frag)->segment.seg_addr.pval +
              mca_btl_openib_component.eager_limit);
 
         MCA_BTL_OPENIB_RDMA_MAKE_REMOTE(frag->ftr);

diff --git a/opal/mca/btl/openib/btl_openib_endpoint.h b/opal/mca/btl/openib/btl_openib_endpoint.h
@@ -546,13 +546,13 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep,
         mca_btl_openib_send_frag_t *frag, const bool rdma, int do_signal)
 {
     mca_btl_openib_module_t *openib_btl = ep->endpoint_btl;
-    mca_btl_openib_segment_t *seg = &to_base_frag(frag)->segment;
+    mca_btl_base_segment_t *seg = &to_base_frag(frag)->segment;
     struct ibv_sge *sg = &to_com_frag(frag)->sg_entry;
     struct ibv_send_wr *sr_desc = &to_out_frag(frag)->sr_desc;
     struct ibv_send_wr *bad_wr;
     int qp = to_base_frag(frag)->base.order;
 
-    sg->length = seg->base.seg_len + sizeof(mca_btl_openib_header_t) +
+    sg->length = seg->seg_len + sizeof(mca_btl_openib_header_t) +
         (rdma ? sizeof(mca_btl_openib_footer_t) : 0) + frag->coalesced_length;
 
     sr_desc->send_flags = ib_send_flags(sg->length, &(ep->qps[qp]), do_signal);

diff --git a/opal/mca/btl/openib/btl_openib_failover.c b/opal/mca/btl/openib/btl_openib_failover.c
@@ -1,7 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2010-2011 Oracle and/or its affiliates.  All rights reserved.
  * Copyright (c) 2011      NVIDIA Corporation.  All rights reserved.
- * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2012-2015 Los Alamos National Security, LLC.  All rights
  *                         reserved. 
  * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
@@ -678,12 +679,12 @@ static void mca_btl_openib_endpoint_notify(mca_btl_base_endpoint_t* endpoint, ui
     to_base_frag(frag)->base.des_cbdata = NULL;
     to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
     to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
-    to_base_frag(frag)->segment.base.seg_len =
+    to_base_frag(frag)->segment.seg_len =
         sizeof(mca_btl_openib_broken_connection_header_t);
     to_com_frag(frag)->endpoint = newep;
 
     frag->hdr->tag = MCA_BTL_TAG_IB;
-    bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
+    bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
     bc_hdr->control.type = type;
     bc_hdr->lid = endpoint->endpoint_btl->port_info.lid;
     bc_hdr->subnet_id = endpoint->endpoint_btl->port_info.subnet_id;
@@ -720,10 +721,10 @@ static void dump_local_rdma_frags(mca_btl_openib_endpoint_t * endpoint) {
 
         frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
                size + sizeof(mca_btl_openib_footer_t));
-        to_base_frag(frag)->segment.base.seg_addr.pval =
+        to_base_frag(frag)->segment.seg_addr.pval =
                ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
 
-        chdr = to_base_frag(frag)->segment.base.seg_addr.pval;
+        chdr = to_base_frag(frag)->segment.seg_addr.pval;
         if ((MCA_BTL_TAG_IB == frag->hdr->tag) &&
             (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type)) {
             opal_output(0, "tag[%d] is credit message", i);

diff --git a/opal/mca/btl/openib/btl_openib_frag.c b/opal/mca/btl/openib/btl_openib_frag.c
@@ -62,7 +62,6 @@ static void com_constructor(mca_btl_openib_com_frag_t *frag)
 
     if(reg) {
         frag->sg_entry.lkey = reg->mr->lkey;
-        base_frag->segment.key = reg->mr->lkey;
     }
     frag->n_wqes_inflight = 0;
 }
@@ -71,7 +70,7 @@ static void out_constructor(mca_btl_openib_out_frag_t *frag)
 {
     mca_btl_openib_frag_t *base_frag = to_base_frag(frag);
 
-    base_frag->base.des_segments = &base_frag->segment.base;
+    base_frag->base.des_segments = &base_frag->segment;
     base_frag->base.des_segment_count = 1;
 
     frag->sr_desc.wr_id = (uint64_t)(uintptr_t)frag;
@@ -86,7 +85,7 @@ static void in_constructor(mca_btl_openib_in_frag_t *frag)
 {
     mca_btl_openib_frag_t *base_frag = to_base_frag(frag);
 
-    base_frag->base.des_segments = &base_frag->segment.base;
+    base_frag->base.des_segments = &base_frag->segment;
     base_frag->base.des_segment_count = 1;
 }
 
@@ -101,7 +100,7 @@ static void send_constructor(mca_btl_openib_send_frag_t *frag)
         (((unsigned char*)base_frag->base.super.ptr) +
         sizeof(mca_btl_openib_header_coalesced_t) +
         sizeof(mca_btl_openib_control_header_t));
-    base_frag->segment.base.seg_addr.pval = frag->hdr + 1;
+    base_frag->segment.seg_addr.pval = frag->hdr + 1;
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
     frag->coalesced_length = 0;
     OBJ_CONSTRUCT(&frag->coalesced_frags, opal_list_t);
@@ -114,7 +113,7 @@ static void recv_constructor(mca_btl_openib_recv_frag_t *frag)
     base_frag->type = MCA_BTL_OPENIB_FRAG_RECV;
 
     frag->hdr = (mca_btl_openib_header_t*)base_frag->base.super.ptr;
-    base_frag->segment.base.seg_addr.pval =
+    base_frag->segment.seg_addr.pval =
         ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
 
@@ -129,7 +128,7 @@ static void send_control_constructor(mca_btl_openib_send_control_frag_t *frag)
     to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_CONTROL;
     /* adjusting headers because there is no coalesce header in control messages */
     frag->hdr = frag->chdr;
-    to_base_frag(frag)->segment.base.seg_addr.pval = frag->hdr + 1;
+    to_base_frag(frag)->segment.seg_addr.pval = frag->hdr + 1;
     to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
 }
 
@@ -158,7 +157,7 @@ static void coalesced_constructor(mca_btl_openib_coalesced_frag_t *frag)
 
     base_frag->type = MCA_BTL_OPENIB_FRAG_COALESCED;
 
-    base_frag->base.des_segments = &base_frag->segment.base;
+    base_frag->base.des_segments = &base_frag->segment;
     base_frag->base.des_segment_count = 1;
 }
 

diff --git a/opal/mca/btl/openib/btl_openib_frag.h b/opal/mca/btl/openib/btl_openib_frag.h
@@ -284,16 +284,10 @@ typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t;
 /**
  * IB fragment derived type.
  */
-
-typedef struct mca_btl_openib_segment_t {
-    mca_btl_base_segment_t base;
-    uint32_t key;
-} mca_btl_openib_segment_t;
-
 /* base openib frag */
 typedef struct mca_btl_openib_frag_t {
     mca_btl_base_descriptor_t base;
-    mca_btl_openib_segment_t segment;
+    mca_btl_base_segment_t segment;
     mca_btl_openib_frag_type_t type;
     opal_free_list_t* list;
 } mca_btl_openib_frag_t;

diff --git a/opal/mca/btl/openib/connect/btl_openib_connect_base.c b/opal/mca/btl/openib/connect/btl_openib_connect_base.c
@@ -448,7 +448,6 @@ int opal_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
 
     /* Copy the lkey where it needs to go */
     endpoint->endpoint_cts_frag.super.sg_entry.lkey =
-        endpoint->endpoint_cts_frag.super.super.segment.key =
         endpoint->endpoint_cts_mr->lkey;
     endpoint->endpoint_cts_frag.super.sg_entry.length = length;