When frags are allocated from mpool by free_list the frag structure is also

allocated from mpool memory (which is registered memory for RDMA transports)
This is not a problem for a small jobs, but for a big number of ranks an
amount of waisted memory is big.

This commit was SVN r13921.

Author: Gleb Natapov

ompi/class/ompi_free_list.c

@@ -20,26 +20,11 @@
 #include "ompi_config.h"
 
 #include "ompi/class/ompi_free_list.h"
+#include "opal/include/opal/align.h"
 #include "opal/sys/cache.h"
 #include "opal/util/output.h"
 #include "ompi/mca/mpool/mpool.h"
 
-static inline size_t align_to(size_t val, size_t alignment);
-static inline size_t align_to(size_t val, size_t alignment)
-{
-    size_t mod;
-   
-    if(0 == alignment)
-        return val;
-
-    mod = val % alignment;
-    
-    if(mod)
-        val += (alignment - mod);
-
-    return val;
-}
-
 static void ompi_free_list_construct(ompi_free_list_t* fl);
 static void ompi_free_list_destruct(ompi_free_list_t* fl);
 
@@ -49,6 +34,7 @@ OBJ_CLASS_INSTANCE(ompi_free_list_t, opal_atomic_lifo_t,
 struct ompi_free_list_memory_t {
     opal_list_item_t super;
     mca_mpool_base_registration_t *registration;
+    void *base_ptr;
 };
 typedef struct ompi_free_list_memory_t ompi_free_list_memory_t;
 static OBJ_CLASS_INSTANCE(ompi_free_list_memory_t,
@@ -69,7 +55,6 @@ static void ompi_free_list_construct(ompi_free_list_t* fl)
     fl->fl_num_waiting = 0;
     fl->fl_elem_size = sizeof(ompi_free_list_item_t);
     fl->fl_elem_class = OBJ_CLASS(ompi_free_list_item_t);
-    fl->fl_header_space = 0;
     fl->fl_alignment = 0;
     fl->fl_mpool = 0;
     OBJ_CONSTRUCT(&(fl->fl_allocations), opal_list_t);
@@ -78,6 +63,7 @@ static void ompi_free_list_construct(ompi_free_list_t* fl)
 static void ompi_free_list_destruct(ompi_free_list_t* fl)
 {
     opal_list_item_t *item;
+    ompi_free_list_memory_t *fl_mem;
 
 #if 0 && OMPI_ENABLE_DEBUG
     if(opal_list_get_size(&fl->super) != fl->fl_num_allocated) {
@@ -87,21 +73,15 @@ static void ompi_free_list_destruct(ompi_free_list_t* fl)
     }
 #endif
 
-    if (NULL != fl->fl_mpool) {
-        ompi_free_list_memory_t *fl_mem;
-
-        while (NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
-            /* destruct the item (we constructed it), then free the memory chunk */
-            OBJ_DESTRUCT(item);
-            fl_mem = (ompi_free_list_memory_t*) item;
-            fl->fl_mpool->mpool_free(fl->fl_mpool, item, fl_mem->registration);
-        }
-    } else {
-        while (NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
-            /* destruct the item (we constructed it), then free the memory chunk */
-            OBJ_DESTRUCT(item);
-            free(item);
+    while(NULL != (item = opal_list_remove_first(&(fl->fl_allocations)))) {
+        fl_mem = (ompi_free_list_memory_t*)item;
+        if(fl->fl_mpool != NULL) {
+            fl->fl_mpool->mpool_free(fl->fl_mpool, fl_mem->base_ptr,
+                    fl_mem->registration);
         }
+       /* destruct the item (we constructed it), then free the memory chunk */
+        OBJ_DESTRUCT(item);
+        free(item);
     }
 
     OBJ_DESTRUCT(&fl->fl_allocations);
@@ -112,14 +92,17 @@ static void ompi_free_list_destruct(ompi_free_list_t* fl)
 int ompi_free_list_init_ex(
     ompi_free_list_t *flist,
     size_t elem_size,
-    size_t header_space,
     size_t alignment,
     opal_class_t* elem_class,
     int num_elements_to_alloc,
     int max_elements_to_alloc,
     int num_elements_per_alloc,
     mca_mpool_base_module_t* mpool)
 {
+    /* alignment must be more than zero and power of two */
+    if(alignment <= 1 || (alignment & (alignment - 1)))
+        return OMPI_ERROR;
+
     if(elem_size > flist->fl_elem_size)
         flist->fl_elem_size = elem_size;
     if(elem_class)
@@ -128,61 +111,75 @@ int ompi_free_list_init_ex(
     flist->fl_num_allocated = 0;
     flist->fl_num_per_alloc = num_elements_per_alloc;
     flist->fl_mpool = mpool;
-    flist->fl_header_space = header_space;
     flist->fl_alignment = alignment;
-    flist->fl_elem_size = align_to(flist->fl_elem_size, flist->fl_alignment);
     if(num_elements_to_alloc)
         return ompi_free_list_grow(flist, num_elements_to_alloc);
     return OMPI_SUCCESS;
 }
 
 int ompi_free_list_grow(ompi_free_list_t* flist, size_t num_elements)
 {
-    unsigned char* ptr;
+    unsigned char *ptr, *mpool_alloc_ptr = NULL;
     ompi_free_list_memory_t *alloc_ptr;
-    size_t i, alloc_size;
-    mca_mpool_base_registration_t* user_out = NULL; 
+    size_t i, alloc_size, head_size, elem_size = 0;
+    mca_mpool_base_registration_t *reg = NULL;
 
-    if (flist->fl_max_to_alloc > 0)
-        if (flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
+    if(flist->fl_max_to_alloc > 0)
+        if(flist->fl_num_allocated + num_elements > flist->fl_max_to_alloc)
             num_elements = flist->fl_max_to_alloc - flist->fl_num_allocated;
 
-    if (num_elements == 0)
+    if(num_elements == 0)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
-    alloc_size = num_elements * flist->fl_elem_size +
-        sizeof(ompi_free_list_memory_t) + flist->fl_header_space +
+    head_size = (NULL == flist->fl_mpool) ? flist->fl_elem_size:
+        flist->fl_elem_class->cls_sizeof;
+    head_size = OPAL_ALIGN(head_size, flist->fl_alignment, size_t);
+
+    /* calculate head allocation size */
+    alloc_size = num_elements * head_size + sizeof(ompi_free_list_memory_t) +
         flist->fl_alignment;
 
-    if (NULL != flist->fl_mpool)
-        alloc_ptr = (ompi_free_list_memory_t*)flist->fl_mpool->mpool_alloc(flist->fl_mpool, 
-                                                                           alloc_size, 0, MCA_MPOOL_FLAGS_CACHE_BYPASS, &user_out);
-    else
-        alloc_ptr = (ompi_free_list_memory_t*)malloc(alloc_size);
+    alloc_ptr = (ompi_free_list_memory_t*)malloc(alloc_size);
 
     if(NULL == alloc_ptr)
         return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
 
-    /* make the alloc_ptr a list item, save the chunk in the allocations list, and
-       have ptr point to memory right after the list item structure */
-    OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_memory_t);
-    opal_list_append(&(flist->fl_allocations), (opal_list_item_t*) alloc_ptr);
+    /* allocate the rest from the mpool */
+    if(flist->fl_mpool != NULL) {
+        elem_size = OPAL_ALIGN(flist->fl_elem_size -
+                flist->fl_elem_class->cls_sizeof, flist->fl_alignment, size_t);
+        if(elem_size != 0) {
+            mpool_alloc_ptr = flist->fl_mpool->mpool_alloc(flist->fl_mpool,
+                   num_elements * elem_size, flist->fl_alignment,
+                   MCA_MPOOL_FLAGS_CACHE_BYPASS, &reg);
+            if(NULL == mpool_alloc_ptr) {
+                free(alloc_ptr);
+                return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
+            }
+        }
+    }
 
-    alloc_ptr->registration = user_out;
+    /* make the alloc_ptr a list item, save the chunk in the allocations list,
+     * and have ptr point to memory right after the list item structure */
+    OBJ_CONSTRUCT(alloc_ptr, ompi_free_list_memory_t);
+    opal_list_append(&(flist->fl_allocations), (opal_list_item_t*)alloc_ptr);
 
-    ptr = (unsigned char*) alloc_ptr + sizeof(ompi_free_list_memory_t);
+    alloc_ptr->registration = reg;
+    alloc_ptr->base_ptr = mpool_alloc_ptr;
 
-    ptr = (unsigned char*)(align_to((size_t)ptr + flist->fl_header_space,
-                flist->fl_alignment) - flist->fl_header_space);
+    ptr = (unsigned char*)alloc_ptr + sizeof(ompi_free_list_memory_t);
+    ptr = OPAL_ALIGN_PTR(ptr, flist->fl_alignment, unsigned char*);
 
     for(i=0; i<num_elements; i++) {
         ompi_free_list_item_t* item = (ompi_free_list_item_t*)ptr;
-        item->user_data = user_out; 
+        item->registration = reg;
+        item->ptr = mpool_alloc_ptr;
 
         OBJ_CONSTRUCT_INTERNAL(item, flist->fl_elem_class);
 
         opal_atomic_lifo_push(&(flist->super), &(item->super));
-        ptr += flist->fl_elem_size;
+        ptr += head_size;
+        mpool_alloc_ptr += elem_size;
     }
 
     flist->fl_num_allocated += num_elements;

ompi/class/ompi_free_list.h

@@ -40,7 +40,6 @@ struct ompi_free_list_t
     size_t fl_num_per_alloc;
     size_t fl_num_waiting;
     size_t fl_elem_size;
-    size_t fl_header_space;
     size_t fl_alignment;
     opal_class_t* fl_elem_class;
     struct mca_mpool_base_module_t* fl_mpool;
@@ -51,10 +50,12 @@ struct ompi_free_list_t
 typedef struct ompi_free_list_t ompi_free_list_t;
 OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_free_list_t);
 
+struct mca_mpool_base_registration_t;
 struct ompi_free_list_item_t
 { 
     opal_list_item_t super; 
-    void* user_data; 
+    struct mca_mpool_base_registration_t *registration;
+    void *ptr;
 }; 
 typedef struct ompi_free_list_item_t ompi_free_list_item_t; 
 
@@ -75,7 +76,6 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_free_list_item_t);
 OMPI_DECLSPEC int ompi_free_list_init_ex(
     ompi_free_list_t *free_list, 
     size_t element_size,
-    size_t header_size,
     size_t alignment,
     opal_class_t* element_class,
     int num_elements_to_alloc,
@@ -92,7 +92,7 @@ static inline int ompi_free_list_init(
     int num_elements_per_alloc,
     struct mca_mpool_base_module_t* mpool)
 {
-    return ompi_free_list_init_ex(free_list, element_size, 0, CACHE_LINE_SIZE,
+    return ompi_free_list_init_ex(free_list, element_size, CACHE_LINE_SIZE,
             element_class, num_elements_to_alloc, max_elements_to_alloc,
             num_elements_per_alloc, mpool);
 }

ompi/mca/btl/gm/btl_gm_frag.c

@@ -29,7 +29,7 @@ do {                                             \
 
 static void mca_btl_gm_frag_eager_constructor(mca_btl_gm_frag_t* frag) 
 { 
-    frag->hdr = (mca_btl_base_header_t*)(frag + 1);
+    frag->hdr = (mca_btl_base_header_t*)frag->base.super.ptr;
     frag->segment.seg_addr.pval = (unsigned char*)(frag->hdr + 1); 
     frag->segment.seg_len = mca_btl_gm_module.super.btl_eager_limit - sizeof(mca_btl_base_header_t);
     frag->size = mca_btl_gm_component.gm_eager_frag_size;

ompi/mca/btl/mvapi/btl_mvapi_component.c

@@ -583,7 +583,10 @@ mca_btl_base_module_t** mca_btl_mvapi_component_init(int *num_btl_modules,
             2*MCA_BTL_IB_FRAG_ALIGN; 
 
         mvapi_btl->eager_rdma_frag_size =
-            length & ~(2 * MCA_BTL_IB_FRAG_ALIGN - 1);
+            (sizeof(mca_btl_mvapi_header_t) +
+             sizeof(mca_btl_mvapi_footer_t) +
+             mvapi_btl->super.btl_eager_limit +
+             2*MCA_BTL_IB_FRAG_ALIGN) & ~(2 * MCA_BTL_IB_FRAG_ALIGN - 1);
 
         ompi_free_list_init(&mvapi_btl->send_free_eager,
                             length, 

ompi/mca/btl/mvapi/btl_mvapi_eager_rdma.h

@@ -18,9 +18,11 @@ extern "C" {
 #endif
 
 struct mca_btl_mvapi_reg_t;
+struct mca_btl_mvapi_frag_t;
 
 struct mca_btl_mvapi_eager_rdma_local_t {
 	ompi_ptr_t base; /**< buffer for RDMAing eager messages */
+    struct mca_btl_mvapi_frag_t *frags;
 	struct mca_btl_mvapi_reg_t *reg;
 	uint16_t head; /**< RDMA buffer to poll */
     uint16_t tail; /**< Needed for credit managment */
@@ -73,9 +75,7 @@ typedef struct mca_btl_mvapi_eager_rdma_remote_t mca_btl_mvapi_eager_rdma_remote
                             }while (0)
 
 #define MCA_BTL_MVAPI_GET_LOCAL_RDMA_FRAG(E, I)                         \
-            (mca_btl_mvapi_frag_t*)                                     \
-            ((char*)(E)->eager_rdma_local.base.pval +                   \
-            (I) * (E)->endpoint_btl->eager_rdma_frag_size)
+            (&(E)->eager_rdma_local.frags[(I)])
 
 #define MCA_BTL_MVAPI_RDMA_NEXT_INDEX(I) do {                       \
                             (I) = ((I) + 1) %                        \

ompi/mca/btl/mvapi/btl_mvapi_endpoint.c

@@ -165,10 +165,9 @@ static inline int mca_btl_mvapi_endpoint_post_send(
 #endif
         frag->desc.sr_desc.r_key = (VAPI_rkey_t)endpoint->eager_rdma_remote.rkey;
         frag->desc.sr_desc.remote_addr = (VAPI_virt_addr_t)
-            endpoint->eager_rdma_remote.base.lval +
+            (uintptr_t)endpoint->eager_rdma_remote.base.pval +
             endpoint->eager_rdma_remote.head *
             mvapi_btl->eager_rdma_frag_size +
-            sizeof(mca_btl_mvapi_frag_t) +
             sizeof(mca_btl_mvapi_header_t) +
             frag->size +
             sizeof(mca_btl_mvapi_footer_t);
@@ -1223,25 +1222,34 @@ void mca_btl_mvapi_endpoint_connect_eager_rdma(
 {
     mca_btl_mvapi_module_t* mvapi_btl = endpoint->endpoint_btl;
     char *buf;
+    mca_btl_mvapi_recv_frag_eager_t *headers_buf;
     unsigned int i;
 
     OPAL_THREAD_LOCK(&endpoint->eager_rdma_local.lock);
     if (endpoint->eager_rdma_local.base.pval)
         goto unlock_rdma_local;
 
+    headers_buf = (mca_btl_mvapi_recv_frag_eager_t*)
+        malloc(sizeof(mca_btl_mvapi_recv_frag_eager_t) *
+                mca_btl_mvapi_component.eager_rdma_num);
+
+    if(NULL == headers_buf)
+        goto unlock_rdma_local;
+
     buf = mvapi_btl->super.btl_mpool->mpool_alloc(mvapi_btl->super.btl_mpool,
             mvapi_btl->eager_rdma_frag_size * 
             mca_btl_mvapi_component.eager_rdma_num, 0,
             MCA_MPOOL_FLAGS_CACHE_BYPASS,
             (mca_mpool_base_registration_t**)&endpoint->eager_rdma_local.reg);
 
     if(!buf)
-       goto unlock_rdma_local;
+       goto free_headers_buf;
 
     for(i = 0; i < mca_btl_mvapi_component.eager_rdma_num; i++) {
-        ompi_free_list_item_t *item = (ompi_free_list_item_t *)(buf +
-                i*mvapi_btl->eager_rdma_frag_size);
-        item->user_data = (void*)endpoint->eager_rdma_local.reg;
+        ompi_free_list_item_t *item;
+        item = (ompi_free_list_item_t *)&headers_buf[i];
+        item->registration = (void*)endpoint->eager_rdma_local.reg;
+        item->ptr = buf + i * mvapi_btl->eager_rdma_frag_size;
         OBJ_CONSTRUCT(item, mca_btl_mvapi_recv_frag_eager_t);
         ((mca_btl_mvapi_frag_t*)item)->endpoint = endpoint;
         ((mca_btl_mvapi_frag_t*)item)->type = MCA_BTL_MVAPI_FRAG_EAGER_RDMA;
@@ -1253,6 +1261,7 @@ void mca_btl_mvapi_endpoint_connect_eager_rdma(
       goto cleanup;
 
     endpoint->eager_rdma_local.base.pval = buf;
+    endpoint->eager_rdma_local.frags = headers_buf;
     mvapi_btl->eager_rdma_buffers_count++;
     if (mca_btl_mvapi_endpoint_send_eager_rdma(endpoint) == 0) {
         OPAL_THREAD_UNLOCK(&mvapi_btl->eager_rdma_lock);
@@ -1262,13 +1271,16 @@ void mca_btl_mvapi_endpoint_connect_eager_rdma(
 
     mvapi_btl->eager_rdma_buffers_count--;
     endpoint->eager_rdma_local.base.pval = NULL;
+    endpoint->eager_rdma_local.frags = NULL;
     orte_pointer_array_set_item(mvapi_btl->eager_rdma_buffers,
             endpoint->eager_rdma_index, NULL);
 
 cleanup:
     OPAL_THREAD_UNLOCK(&mvapi_btl->eager_rdma_lock);
     mvapi_btl->super.btl_mpool->mpool_free(mvapi_btl->super.btl_mpool,
             buf, (mca_mpool_base_registration_t*)endpoint->eager_rdma_local.reg);
+free_headers_buf:
+    free(headers_buf);
 unlock_rdma_local:
     OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
 }

ompi/mca/btl/mvapi/btl_mvapi_frag.c

@@ -23,8 +23,8 @@
 static void mca_btl_mvapi_frag_common_constructor( mca_btl_mvapi_frag_t* frag) 
 {
     mca_btl_mvapi_reg_t* mem_hndl =
-        (mca_btl_mvapi_reg_t*)frag->base.super.user_data;
-    frag->hdr = (mca_btl_mvapi_header_t*) (frag+1);  /* initialize btl header to start at end of frag */ 
+        (mca_btl_mvapi_reg_t*)frag->base.super.registration;
+    frag->hdr = (mca_btl_mvapi_header_t*)frag->base.super.ptr;
     frag->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_mvapi_header_t);  
     /* init the segment address to start after the btl header */