Skip to content

btl-portals4: fix bugs in the Portals4 BTL after the port to BTL3 interface #545

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 22, 2015
Merged

btl-portals4: fix bugs in the Portals4 BTL after the port to BTL3 interface #545

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bca2522
btl-portals4: fix send failure for messages larger than eager_size
tkordenbrock Apr 10, 2015
b1cef6c
btl-portals4: remove unused code path
tkordenbrock Apr 14, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 44 additions & 84 deletions opal/mca/btl/portals4/btl_portals4.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -324,90 +324,6 @@ mca_btl_portals4_prepare_src(struct mca_btl_base_module_t* btl_base,

frag->segments[0].base.seg_len = max_data + reserve;
frag->base.des_segment_count = 1;

} else {
/* no need to pack - rdma operation out of user's buffer */
ptl_me_t me;

/* reserve space in the event queue for rdma operations immediately */
while (OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, 1) >
portals4_btl->portals_max_outstanding_ops) {
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "Call to mca_btl_portals4_component_progress (1)\n"));
mca_btl_portals4_component_progress();
}

OPAL_BTL_PORTALS4_FRAG_ALLOC_USER(portals4_btl, frag);
if (NULL == frag){
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_prepare_src: Incrementing portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));

iov.iov_len = max_data;
iov.iov_base = NULL;

ret = opal_convertor_pack(convertor, &iov, &iov_count, &max_data );
if ( OPAL_UNLIKELY(ret < 0) ) {
OPAL_BTL_PORTALS4_FRAG_RETURN_USER(portals4_btl, frag);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}

frag->segments[0].base.seg_len = max_data;
frag->segments[0].base.seg_addr.pval = iov.iov_base;
frag->segments[0].key = OPAL_THREAD_ADD64(&(portals4_btl->portals_rdma_key), 1);
frag->base.des_segment_count = 1;

/* either a put or get. figure out which later */
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"rdma src posted for frag 0x%lx, callback 0x%lx, bits %"PRIu64", flags say %d" ,
(unsigned long) frag,
(unsigned long) frag->base.des_cbfunc,
frag->segments[0].key, flags));

/* create a match entry */
me.start = frag->segments[0].base.seg_addr.pval;
me.length = frag->segments[0].base.seg_len;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;

if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = peer->ptl_proc.rank;
} else {
me.match_id.phys.nid = peer->ptl_proc.phys.nid;
me.match_id.phys.pid = peer->ptl_proc.phys.pid;
}
me.match_bits = frag->segments[0].key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;

ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
frag,
&(frag->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_BTL_PORTALS4_FRAG_RETURN_USER(portals4_btl, frag);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (prepare_src) frag=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)frag, frag->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
}

frag->base.des_segments = &frag->segments[0].base;
Expand All @@ -425,6 +341,8 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
{
struct mca_btl_portals4_module_t *portals4_btl = (struct mca_btl_portals4_module_t*) btl_base;
mca_btl_base_registration_handle_t *handle = NULL;
ptl_me_t me;
int ret;

handle = (mca_btl_base_registration_handle_t *)malloc(sizeof(mca_btl_base_registration_handle_t));
if (!handle) {
Expand All @@ -437,6 +355,48 @@ mca_btl_portals4_register_mem(mca_btl_base_module_t *btl_base,
"mca_btl_portals4_register_mem NI=%d base=%p size=%ld handle=%p key=%ld\n",
portals4_btl->interface_num, base, size, (void *)handle, handle->key));

if (MCA_BTL_FLAGS_PUT == flags) {
/* create a match entry */
me.start = base;
me.length = size;
me.ct_handle = PTL_CT_NONE;
me.min_free = 0;
me.uid = PTL_UID_ANY;
me.options = PTL_ME_OP_GET | PTL_ME_USE_ONCE |
PTL_ME_EVENT_LINK_DISABLE |
PTL_ME_EVENT_COMM_DISABLE |
PTL_ME_EVENT_UNLINK_DISABLE;

if (mca_btl_portals4_component.use_logical) {
me.match_id.rank = endpoint->ptl_proc.rank;
} else {
me.match_id.phys.nid = endpoint->ptl_proc.phys.nid;
me.match_id.phys.pid = endpoint->ptl_proc.phys.pid;
}
me.match_bits = handle->key;
me.ignore_bits = BTL_PORTALS4_PROTOCOL_MASK |
BTL_PORTALS4_CONTEXT_MASK |
BTL_PORTALS4_SOURCE_MASK;
me.ignore_bits = 0;

ret = PtlMEAppend(portals4_btl->portals_ni_h,
portals4_btl->recv_idx,
&me,
PTL_PRIORITY_LIST,
handle,
&(handle->me_h));
if (PTL_OK != ret) {
opal_output_verbose(1, opal_btl_base_framework.framework_output,
"%s:%d: PtlMEAppend failed: %d\n",
__FILE__, __LINE__, ret);
OPAL_THREAD_ADD32(&portals4_btl->portals_outstanding_ops, -1);
return NULL;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"PtlMEAppend (mca_btl_portals4_register_mem) handle=%p, me_h=%d start=%p length=%ld rank=%x nid=%x pid=%x match_bits=%lx\n",
(void *)handle, handle->me_h, me.start, me.length,
me.match_id.rank, me.match_id.phys.nid, me.match_id.phys.pid, me.match_bits));
}
return handle;
}

Expand Down
2 changes: 2 additions & 0 deletions opal/mca/btl/portals4/btl_portals4.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,8 @@ int mca_btl_portals4_get_error(int ptl_error);
struct mca_btl_base_registration_handle_t {
/** Portals4 match bits */
ptl_match_bits_t key;
/** Portals4 me_h */
ptl_handle_me_t me_h;
};

/*
Expand Down
12 changes: 7 additions & 5 deletions opal/mca/btl/portals4/btl_portals4_rdma.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,8 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_prepare_src: Incrementing portals_outstanding_ops=%d\n", portals4_btl->portals_outstanding_ops));

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output,
"mca_btl_portals4_get frag=%p\n", (void *)frag));
"mca_btl_portals4_get: Incrementing portals_outstanding_ops=%d frag=%p",
portals4_btl->portals_outstanding_ops, (void *)frag));

frag->rdma_cb.func = cbfunc;
frag->rdma_cb.context = cbcontext;
Expand Down Expand Up @@ -101,6 +99,10 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
frag->match_bits = remote_handle->key;
frag->length = md.length;
frag->peer_proc = btl_peer->ptl_proc;

OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));

ret = PtlGet(frag->md_h,
0,
md.length,
Expand All @@ -117,7 +119,7 @@ mca_btl_portals4_get(struct mca_btl_base_module_t* btl_base,
frag->md_h = PTL_INVALID_HANDLE;
return OPAL_ERROR;
}
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
OPAL_OUTPUT_VERBOSE((90, opal_btl_base_framework.framework_output, "SUCCESS: PtlGet start=%p length=%ld nid=%x pid=%x match_bits=%lx\n",
md.start, md.length, btl_peer->ptl_proc.phys.nid, btl_peer->ptl_proc.phys.pid, frag->match_bits));

return OPAL_SUCCESS;
Expand Down