diff --git a/ompi/mca/osc/portals4/osc_portals4.h b/ompi/mca/osc/portals4/osc_portals4.h index c80b2b4a58f..b35c0ed9053 100644 --- a/ompi/mca/osc/portals4/osc_portals4.h +++ b/ompi/mca/osc/portals4/osc_portals4.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2011-2017 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -23,6 +23,8 @@ #define REQ_OSC_TABLE_ID 4 +#define OSC_PORTALS4_IOVEC_MAX 64 + #define OSC_PORTALS4_MB_DATA 0x0000000000000000ULL #define OSC_PORTALS4_MB_CONTROL 0x1000000000000000ULL @@ -95,6 +97,11 @@ struct ompi_osc_portals4_module_t { int64_t opcount; ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */ + ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */ + ptl_handle_md_t origin_iovec_md_h; /* memory descriptor describing a noncontiguous region in this window */ + ptl_iovec_t *result_iovec_list; /* list of memory segments that compose the noncontiguous region */ + ptl_handle_md_t result_iovec_md_h; /* memory descriptor describing a noncontiguous region in this window */ + ptl_size_t atomic_max; /* max size of atomic messages. Will guarantee ordering IF ordering requested */ ptl_size_t fetch_atomic_max; /* max size of fetchatomic messages. Will guarantee ordering IF ordering requested */ diff --git a/ompi/mca/osc/portals4/osc_portals4_comm.c b/ompi/mca/osc/portals4/osc_portals4_comm.c index b6bb3562c66..3b197f9708c 100644 --- a/ompi/mca/osc/portals4/osc_portals4_comm.c +++ b/ompi/mca/osc/portals4/osc_portals4_comm.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2011-2017 Sandia National Laboratories. All rights reserved. * Copyright (c) 2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. @@ -179,7 +179,7 @@ ompi_osc_portals4_get_dt(struct ompi_datatype_t *dt, ptl_datatype_t *ptl_dt) } static ptl_size_t -number_of_fragment(ptl_size_t length, ptl_size_t maxlength) +number_of_fragments(ptl_size_t length, ptl_size_t maxlength) { ptl_size_t nb_frag = length == 0 ? 1 : (length - 1) / maxlength + 1; OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, @@ -187,87 +187,1833 @@ number_of_fragment(ptl_size_t length, ptl_size_t maxlength) return nb_frag; } +/* put in segments no larger than segment_length */ static int -splittedPtlPut(ptl_handle_md_t md_h, - ptl_size_t loc_offset, - ptl_size_t length, - ptl_ack_req_t ack_req, - ptl_process_t target_id, - ptl_pt_index_t pt_index, - ptl_match_bits_t match_b, - ptl_size_t rem_offset, - void *usr_ptr, - ptl_hdr_data_t hdr_data) +segmentedPut(int64_t *opcount, + ptl_handle_md_t md_h, + ptl_size_t origin_offset, + ptl_size_t put_length, + ptl_size_t segment_length, + ptl_ack_req_t ack_req, + ptl_process_t target_id, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + ptl_size_t target_offset, + void *user_ptr, + ptl_hdr_data_t hdr_data) { - ptl_size_t length_sent = 0; + int ret; + ptl_size_t bytes_put = 0; + + do { + opal_atomic_add_64(opcount, 1); + + ptl_size_t frag_length = MIN(put_length, segment_length); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "Put size : %lu/%lu, offset:%lu", frag_length, put_length, bytes_put)); + ret = PtlPut(md_h, + origin_offset + bytes_put, + frag_length, + ack_req, + target_id, + pt_index, + match_bits, + target_offset + bytes_put, + user_ptr, + hdr_data); + if (PTL_OK != ret) { + opal_atomic_add_64(opcount, -1); + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlPut failed with return value %d", + __FUNCTION__, __LINE__, ret); + return ret; + } + put_length -= frag_length; + bytes_put += frag_length; + } while (put_length); + return PTL_OK; +} + +/* get in segments no larger than segment_length */ +static int +segmentedGet(int64_t *opcount, + ptl_handle_md_t md_h, + ptl_size_t origin_offset, + ptl_size_t get_length, + ptl_size_t segment_length, + ptl_process_t target_id, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + ptl_size_t target_offset, + void *user_ptr) +{ + int ret; + ptl_size_t bytes_gotten = 0; + + do { + opal_atomic_add_64(opcount, 1); + + ptl_size_t frag_length = MIN(get_length, segment_length); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "Get size : %lu/%lu, offset:%lu", frag_length, get_length, bytes_gotten)); + + ret = PtlGet(md_h, + (ptl_size_t) origin_offset + bytes_gotten, + frag_length, + target_id, + pt_index, + match_bits, + target_offset + bytes_gotten, + user_ptr); + if (PTL_OK != ret) { + opal_atomic_add_64(opcount, -1); + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlGet failed with return value %d", + __FUNCTION__, __LINE__, ret); + return ret; + } + get_length -= frag_length; + bytes_gotten += frag_length; + } while (get_length); + return PTL_OK; +} + +/* atomic op in segments no larger than segment_length */ +static int +segmentedAtomic(int64_t *opcount, + ptl_handle_md_t md_h, + ptl_size_t origin_offset, + ptl_size_t length, + ptl_size_t segment_length, + ptl_process_t target_id, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + ptl_size_t target_offset, + void *user_ptr, + ptl_op_t ptl_op, + ptl_datatype_t ptl_dt) +{ + int ret; + ptl_size_t sent = 0; + + do { + opal_atomic_add_64(opcount, 1); + + ptl_size_t frag_length = MIN(length, segment_length); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "Atomic size : %lu/%lu, offset:%lu", frag_length, length, sent)); + ret = PtlAtomic(md_h, + (ptl_size_t) origin_offset + sent, + frag_length, + PTL_ACK_REQ, + target_id, + pt_index, + match_bits, + target_offset + sent, + user_ptr, + 0, + ptl_op, + ptl_dt); + if (PTL_OK != ret) { + opal_atomic_add_64(opcount, -1); + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlAtomic failed with return value %d", + __FUNCTION__, __LINE__, ret); + return ret; + } + length -= frag_length; + sent += frag_length; + } while (length); + return PTL_OK; +} + +/* atomic op in segments no larger than segment_length */ +static int +segmentedFetchAtomic(int64_t *opcount, + ptl_handle_md_t result_md_h, + ptl_size_t result_offset, + ptl_handle_md_t origin_md_h, + ptl_size_t origin_offset, + ptl_size_t length, + ptl_size_t segment_length, + ptl_process_t target_id, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + ptl_size_t target_offset, + void *user_ptr, + ptl_op_t ptl_op, + ptl_datatype_t ptl_dt) +{ + int ret; + ptl_size_t sent = 0; + + do { + opal_atomic_add_64(opcount, 1); + + ptl_size_t frag_length = MIN(length, segment_length); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "Atomic size : %lu/%lu, offset:%lu", frag_length, length, sent)); + ret = PtlFetchAtomic(result_md_h, + result_offset + sent, + origin_md_h, + origin_offset + sent, + frag_length, + target_id, + pt_index, + match_bits, + target_offset + sent, + user_ptr, + 0, + ptl_op, + ptl_dt); + if (PTL_OK != ret) { + opal_atomic_add_64(opcount, -1); + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlFetchAtomic failed with return value %d", + __FUNCTION__, __LINE__, ret); + return ret; + } + length -= frag_length; + sent += frag_length; + } while (length); + return PTL_OK; +} + +/* swap in segments no larger than segment_length */ +static int +segmentedSwap(int64_t *opcount, + ptl_handle_md_t result_md_h, + ptl_size_t result_offset, + ptl_handle_md_t origin_md_h, + ptl_size_t origin_offset, + ptl_size_t length, + ptl_size_t segment_length, + ptl_process_t target_id, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + ptl_size_t target_offset, + void *user_ptr, + ptl_datatype_t ptl_dt) +{ + int ret; + ptl_size_t sent = 0; + + do { + opal_atomic_add_64(opcount, 1); + + ptl_size_t frag_length = MIN(length, segment_length); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "Swap size : %lu/%lu, offset:%lu", frag_length, length, sent)); + ret = PtlSwap(result_md_h, + result_offset + sent, + origin_md_h, + (ptl_size_t) origin_offset + sent, + frag_length, + target_id, + pt_index, + match_bits, + target_offset + sent, + user_ptr, + 0, + NULL, + PTL_SWAP, + ptl_dt); + if (PTL_OK != ret) { + opal_atomic_add_64(opcount, -1); + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlSwap failed with return value %d", + __FUNCTION__, __LINE__, ret); + return ret; + } + length -= frag_length; + sent += frag_length; + } while (length); + return PTL_OK; +} + +static int +create_iov_list(const void *address, + int count, + ompi_datatype_t *datatype, + ptl_iovec_t **ptl_iovec, + ptl_size_t *ptl_iovec_count) +{ + struct iovec iov[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t convertor; + uint32_t iov_count; + uint32_t iov_index, ptl_iovec_index; + /* needed for opal_convertor_raw but not used */ + size_t size; + int ret; + bool done; + + OBJ_CONSTRUCT(&convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &datatype->super, count, + address, 0, &convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + + *ptl_iovec_count = 0; + ptl_iovec_index = 0; + do { + /* decode segments of the data */ + iov_count = OSC_PORTALS4_IOVEC_MAX; + iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&convertor, iov, &iov_count, &size); + + *ptl_iovec_count += iov_count; + *ptl_iovec = (ptl_iovec_t *)realloc(*ptl_iovec, *ptl_iovec_count * sizeof(ptl_iovec_t)); + + while (iov_index != iov_count) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "adding iov[%d].[%p,%lu] to ptl_iovec", iov_index, iov[iov_index].iov_base, iov[iov_index].iov_len)); + (*ptl_iovec)[ptl_iovec_index].iov_base = iov[iov_index].iov_base; + (*ptl_iovec)[ptl_iovec_index].iov_len = iov[iov_index].iov_len; + + ptl_iovec_index++; + iov_index++; + } + + assert(*ptl_iovec_count == ptl_iovec_index); + } while (!done); + + return OMPI_SUCCESS; + +} + +/* get from a contiguous remote to an iovec local */ +static int +get_to_iovec(ompi_osc_portals4_module_t *module, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + int ret; + size_t size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; + ptl_md_t md; + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + ptl_size_t iovec_count=0; + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + md.start = module->origin_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", + __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + ret = PtlGet(module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d PtlGet() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + return OMPI_SUCCESS; +} + +/* get to an iovec MD from a contiguous target using fragments no larger + * than max_fetch_atomic_size to guarantee atomic writes at the origin */ +static int +atomic_get_to_iovec(ompi_osc_portals4_module_t *module, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + int ret; + size_t size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; + ptl_md_t md; + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + ptl_size_t iovec_count=0; + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + md.start = module->origin_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Get(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", + __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + ret = segmentedGet(&module->opcount, + module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr); + if (PTL_OK != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +/* put from an iovec MD into a contiguous target */ +static int +put_from_iovec(ompi_osc_portals4_module_t *module, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + int ret; + size_t size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; + ptl_md_t md; + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + ptl_size_t iovec_count=0; + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + md.start = module->origin_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, size=%lu, length=%lu, offset=%lu, op_count=%ld)", + __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, size, length, offset, module->opcount)); + ret = PtlPut(module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr, + 0); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d PtlPut() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + return OMPI_SUCCESS; +} + +/* put from an iovec MD into a contiguous target using fragments no larger + * than max_atomic_size to guarantee atomic writes at the target */ +static int +atomic_put_from_iovec(ompi_osc_portals4_module_t *module, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + int ret; + size_t size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; + ptl_md_t md; + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + ptl_size_t iovec_count=0; + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + md.start = module->origin_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", + __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); + ret = segmentedPut(&module->opcount, + module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + module->atomic_max, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + NULL, + 0); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +/* perform atomic operation on iovec local and contiguous remote */ +static int +atomic_from_iovec(ompi_osc_portals4_module_t *module, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + struct ompi_op_t *op, + void *user_ptr) +{ + int ret; + size_t size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; + ptl_md_t md; + ptl_op_t ptl_op; + ptl_datatype_t ptl_dt; + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + ptl_size_t iovec_count=0; + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + md.start = module->origin_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + ret = segmentedAtomic(&module->opcount, + module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + module->atomic_max, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr, + ptl_op, + ptl_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +/* perform atomic operation on iovec local and contiguous remote */ +static int +swap_to_iovec(ompi_osc_portals4_module_t *module, + const void *result_address, + int result_count, + ompi_datatype_t *result_datatype, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + int ret; + size_t size; + ptl_size_t iovec_count=0; + OPAL_PTRDIFF_TYPE length, result_lb, origin_lb, target_lb, extent; + ptl_md_t md; + ptl_datatype_t ptl_dt; + + if (module->result_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->result_iovec_md_h); + free(module->result_iovec_list); + module->result_iovec_md_h = PTL_INVALID_HANDLE; + module->result_iovec_list = NULL; + } + + create_iov_list( + result_address, + result_count, + result_datatype, + &module->result_iovec_list, + &iovec_count); + + md.start = module->result_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->result_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + md.start = module->origin_iovec_list; + md.length = iovec_count; + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + ret = ompi_datatype_get_true_extent(result_datatype, &result_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + ret = segmentedSwap(&module->opcount, + module->result_iovec_md_h, + (ptl_size_t) result_lb, + module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr, + ptl_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +/* perform fetch atomic operation on iovec local and contiguous remote */ +static int +fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module, + const void *result_address, + int result_count, + ompi_datatype_t *result_datatype, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + struct ompi_op_t *op, + void *user_ptr) +{ + int ret; + size_t size; + ptl_size_t iovec_count=0; + OPAL_PTRDIFF_TYPE length, result_lb, origin_lb, target_lb, extent; + ptl_md_t md; + ptl_op_t ptl_op; + ptl_datatype_t ptl_dt; + + if (module->result_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->result_iovec_md_h); + free(module->result_iovec_list); + module->result_iovec_md_h = PTL_INVALID_HANDLE; + module->result_iovec_list = NULL; + } + + create_iov_list( + result_address, + result_count, + result_datatype, + &module->result_iovec_list, + &iovec_count); + + md.start = module->result_iovec_list; + md.length = iovec_count; + if (user_ptr) { + md.options = PTL_IOVEC | PTL_MD_EVENT_SEND_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } else { + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + } + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->result_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->origin_iovec_list = NULL; + } + + create_iov_list( + origin_address, + origin_count, + origin_datatype, + &module->origin_iovec_list, + &iovec_count); + + md.start = module->origin_iovec_list; + md.length = iovec_count; + md.options = PTL_IOVEC | PTL_MD_EVENT_SUCCESS_DISABLE | PTL_MD_EVENT_CT_REPLY | PTL_MD_EVENT_CT_ACK; + md.eq_handle = mca_osc_portals4_component.matching_eq_h; + md.ct_handle = module->ct_h; + ret = PtlMDBind(module->ni_h, &md, &module->origin_iovec_md_h); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d: PtlMDBind(iovec) failed: %d\n", + __FILE__, __LINE__, ret); + return ret; + } + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + ret = ompi_datatype_get_true_extent(result_datatype, &result_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(origin_datatype, &origin_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ret = ompi_datatype_get_true_extent(target_datatype, &target_lb, &extent); + if (OMPI_SUCCESS != ret) { + return ret; + } + ompi_datatype_type_size(origin_datatype, &size); + length = size * origin_count; + + ret = segmentedFetchAtomic(&module->opcount, + module->result_iovec_md_h, + (ptl_size_t) result_lb, + module->origin_iovec_md_h, + (ptl_size_t) origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + user_ptr, + ptl_op, + ptl_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return OMPI_SUCCESS; +} + +/* + * Derived from ompi_osc_rdma_master_noncontig() + */ + +/* put in the largest chunks possible given the noncontiguous restriction */ +static int +put_to_noncontig(int64_t *opcount, + ptl_handle_md_t md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t origin_convertor, target_convertor; + uint32_t origin_iov_count, target_iov_count; + uint32_t origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t origin_size, target_size, rdma_len; + size_t max_rdma_len = mca_osc_portals4_component.ptl_max_msg_size; + int ret; + bool done; + + /* prepare convertors for the source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + origin_iov_index = 0; + origin_iov_count = 0; + + do { + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", + origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlPut(md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + PTL_ACK_REQ, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr, + 0); + if (OPAL_UNLIKELY(PTL_OK != ret)) { + opal_atomic_add_64(opcount, -1); + return ret; + } + + /* adjust io vectors */ + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); + } + } while (!done); + + /* clean up convertors */ + opal_convertor_cleanup (&origin_convertor); + OBJ_DESTRUCT(&origin_convertor); + opal_convertor_cleanup (&target_convertor); + OBJ_DESTRUCT(&target_convertor); + + return OMPI_SUCCESS; +} + +/* put in fragments no larger than max_atomic_size to guarantee atomic writes at the target */ +static int +atomic_put_to_noncontig(ompi_osc_portals4_module_t *module, + ptl_handle_md_t md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t origin_convertor, target_convertor; + uint32_t origin_iov_count, target_iov_count; + uint32_t origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t origin_size, target_size, rdma_len; + size_t max_rdma_len = module->atomic_max; + int ret; + bool done; + + /* prepare convertors for the source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + origin_iov_index = 0; + origin_iov_count = 0; + + do { + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", + origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlPut(md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + PTL_ACK_REQ, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr, + 0); + if (OPAL_UNLIKELY(PTL_OK != ret)) { + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + /* adjust io vectors */ + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); + } + } while (!done); + + return OMPI_SUCCESS; +} + +/* perform atomic operation on (non)contiguous local and noncontiguous remote */ +static int +atomic_to_noncontig(ompi_osc_portals4_module_t *module, + ptl_handle_md_t md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + struct ompi_op_t *op, + void *user_ptr) +{ + struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t origin_convertor, target_convertor; + uint32_t origin_iov_count, target_iov_count; + uint32_t origin_iov_index, target_iov_index; + ptl_op_t ptl_op; + ptl_datatype_t ptl_dt; + /* needed for opal_convertor_raw but not used */ + size_t origin_size, target_size, atomic_len; + int ret; + bool done; + + /* prepare convertors for the source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + origin_iov_index = 0; + origin_iov_count = 0; + + do { + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + atomic_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), module->atomic_max); + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", + origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Atomic", __FUNCTION__, __LINE__)); + ret = PtlAtomic(md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + atomic_len, + PTL_ACK_REQ, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr, + 0, + ptl_op, + ptl_dt); + if (OPAL_UNLIKELY(PTL_OK != ret)) { + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + /* adjust io vectors */ + origin_iovec[origin_iov_index].iov_len -= atomic_len; + target_iovec[target_iov_index].iov_len -= atomic_len; + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + atomic_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + atomic_len); + + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); + } + } while (!done); + + return OMPI_SUCCESS; +} + +/* get from a noncontiguous remote to an (non)contiguous local */ +static int +get_from_noncontig(int64_t *opcount, + ptl_handle_md_t md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t origin_convertor, target_convertor; + uint32_t origin_iov_count, target_iov_count; + uint32_t origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t origin_size, target_size, rdma_len; + size_t max_rdma_len = mca_osc_portals4_component.ptl_max_msg_size; + int ret; + bool done; + + /* prepare convertors for the source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + origin_iov_index = 0; + origin_iov_count = 0; + do { - ptl_size_t length_frag; - int ret; + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } - length_frag = (length > mca_osc_portals4_component.ptl_max_msg_size) ? - mca_osc_portals4_component.ptl_max_msg_size : - length; - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "Put size : %lu/%lu, offset:%lu", length_frag, length, length_sent)); - ret = PtlPut(md_h, - loc_offset + length_sent, - length_frag, - ack_req, - target_id, - pt_index, - match_b, - rem_offset + length_sent, - usr_ptr, - hdr_data); - if (PTL_OK != ret) { - opal_output_verbose(1, ompi_osc_base_framework.framework_output, - "%s:%d PtlPut failed with return value %d", - __FUNCTION__, __LINE__, ret); - return ret; + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", + origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlGet(md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr); + if (OPAL_UNLIKELY(PTL_OK != ret)) { + opal_atomic_add_64(opcount, -1); + return ret; + } + + /* adjust io vectors */ + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } - length -= length_frag; - length_sent += length_frag; - } while (length); - return PTL_OK; + } while (!done); + + return OMPI_SUCCESS; } +/* get from a noncontiguous remote to an (non)contiguous local */ static int -splittedPtlGet(ptl_handle_md_t md_h, - ptl_size_t loc_offset, - ptl_size_t length, - ptl_process_t target_id, - ptl_pt_index_t pt_index, - ptl_match_bits_t match_b, - ptl_size_t rem_offset, - void *usr_ptr) +atomic_get_from_noncontig(ompi_osc_portals4_module_t *module, + ptl_handle_md_t md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) { - ptl_size_t length_submitted = 0; - OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "Get")); + struct iovec origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t origin_convertor, target_convertor; + uint32_t origin_iov_count, target_iov_count; + uint32_t origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t origin_size, target_size, rdma_len; + size_t max_rdma_len = module->fetch_atomic_max; + int ret; + bool done; + + /* prepare convertors for the source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + origin_iov_index = 0; + origin_iov_count = 0; do { - ptl_size_t length_frag; - int ret; - length_frag = (length > mca_osc_portals4_component.ptl_max_msg_size) ? - mca_osc_portals4_component.ptl_max_msg_size : - length; - ret = PtlGet(md_h, - (ptl_size_t) loc_offset + length_submitted, - length_frag, - target_id, - pt_index, - match_b, - rem_offset + length_submitted, - usr_ptr); - if (PTL_OK != ret) { - opal_output_verbose(1, ompi_osc_base_framework.framework_output, - "%s:%d PtlGet failed with return value %d", - __FUNCTION__, __LINE__, ret); - return ret; + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing rdma on contiguous region. local: %p, remote: %p, len: %lu", + origin_iovec[origin_iov_index].iov_base, target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlGet(md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr); + if (OPAL_UNLIKELY(PTL_OK != ret)) { + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + /* adjust io vectors */ + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); } - length -= length_frag; - length_submitted += length_frag; - } while (length); - return PTL_OK; + } while (!done); + + return OMPI_SUCCESS; +} + +/* swap from a noncontiguous remote to an (non)contiguous local */ +static int +swap_from_noncontig(ompi_osc_portals4_module_t *module, + ptl_handle_md_t result_md_h, + const void *result_address, + int result_count, + ompi_datatype_t *result_datatype, + ptl_handle_md_t origin_md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + void *user_ptr) +{ + struct iovec result_iovec[OSC_PORTALS4_IOVEC_MAX], origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t result_convertor, origin_convertor, target_convertor; + uint32_t result_iov_count, origin_iov_count, target_iov_count; + uint32_t result_iov_index, origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t result_size, origin_size, target_size, rdma_len; + size_t max_rdma_len = module->fetch_atomic_max; + ptl_datatype_t ptl_dt; + + int ret; + bool done; + + /* prepare convertors for the result, source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&result_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &result_datatype->super, result_count, + (void*)result_address, 0, &result_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + result_iov_index = 0; + result_iov_count = 0; + origin_iov_index = 0; + origin_iov_count = 0; + + do { + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (result_iov_index == result_iov_count) { + /* decode segments of the target buffer */ + result_iov_count = OSC_PORTALS4_IOVEC_MAX; + result_iov_index = 0; + (void) opal_convertor_raw (&result_convertor, result_iovec, &result_iov_count, &result_size); + } + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != result_iov_count); + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing swap on contiguous region. result: %p origin: %p, target: %p, len: %lu", + result_iovec[result_iov_index].iov_base, + origin_iovec[origin_iov_index].iov_base, + target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlSwap(result_md_h, + (ptl_size_t)result_iovec[result_iov_index].iov_base, + origin_md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr, + 0, + NULL, + PTL_SWAP, + ptl_dt); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlSwap failed with return value %d", + __FUNCTION__, __LINE__, ret); + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + /* adjust io vectors */ + result_iovec[result_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + result_iovec[result_iov_index].iov_base = (void *)((intptr_t) result_iovec[result_iov_index].iov_base + rdma_len); + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + result_iov_index += (0 == result_iovec[result_iov_index].iov_len); + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); + } + } while (!done); + + return OMPI_SUCCESS; +} + +/* swap from a noncontiguous remote to an (non)contiguous local */ +static int +fetch_atomic_from_noncontig(ompi_osc_portals4_module_t *module, + ptl_handle_md_t result_md_h, + const void *result_address, + int result_count, + ompi_datatype_t *result_datatype, + ptl_handle_md_t origin_md_h, + const void *origin_address, + int origin_count, + ompi_datatype_t *origin_datatype, + ptl_process_t peer, + int target_count, + ompi_datatype_t *target_datatype, + size_t offset, + ptl_pt_index_t pt_index, + ptl_match_bits_t match_bits, + struct ompi_op_t *op, + void *user_ptr) +{ + struct iovec result_iovec[OSC_PORTALS4_IOVEC_MAX], origin_iovec[OSC_PORTALS4_IOVEC_MAX], target_iovec[OSC_PORTALS4_IOVEC_MAX]; + opal_convertor_t result_convertor, origin_convertor, target_convertor; + uint32_t result_iov_count, origin_iov_count, target_iov_count; + uint32_t result_iov_index, origin_iov_index, target_iov_index; + /* needed for opal_convertor_raw but not used */ + size_t result_size, origin_size, target_size, rdma_len; + size_t max_rdma_len = module->fetch_atomic_max; + ptl_op_t ptl_op; + ptl_datatype_t ptl_dt; + + int ret; + bool done; + + /* prepare convertors for the result, source and target. these convertors will be used to determine the + * contiguous segments within the source and target. */ + OBJ_CONSTRUCT(&result_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &result_datatype->super, result_count, + (void*)result_address, 0, &result_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&origin_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &origin_datatype->super, origin_count, + (void*)origin_address, 0, &origin_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + OBJ_CONSTRUCT(&target_convertor, opal_convertor_t); + ret = opal_convertor_copy_and_prepare_for_send (ompi_mpi_local_convertor, &target_datatype->super, target_count, + (void *)NULL, 0, &target_convertor); + if (OPAL_UNLIKELY(OMPI_SUCCESS != ret)) { + return ret; + } + + ret = ompi_osc_portals4_get_dt(target_datatype, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + + result_iov_index = 0; + result_iov_count = 0; + origin_iov_index = 0; + origin_iov_count = 0; + + do { + /* decode segments of the remote data */ + target_iov_count = OSC_PORTALS4_IOVEC_MAX; + target_iov_index = 0; + + /* opal_convertor_raw returns done when it has reached the end of the data */ + done = opal_convertor_raw (&target_convertor, target_iovec, &target_iov_count, &target_size); + + /* loop on the target segments until we have exhaused the decoded source data */ + while (target_iov_index != target_iov_count) { + if (result_iov_index == result_iov_count) { + /* decode segments of the target buffer */ + result_iov_count = OSC_PORTALS4_IOVEC_MAX; + result_iov_index = 0; + (void) opal_convertor_raw (&result_convertor, result_iovec, &result_iov_count, &result_size); + } + if (origin_iov_index == origin_iov_count) { + /* decode segments of the target buffer */ + origin_iov_count = OSC_PORTALS4_IOVEC_MAX; + origin_iov_index = 0; + (void) opal_convertor_raw (&origin_convertor, origin_iovec, &origin_iov_count, &origin_size); + } + + /* we already checked that the target was large enough. this should be impossible */ + assert (0 != result_iov_count); + assert (0 != origin_iov_count); + + /* determine how much to transfer in this operation */ + rdma_len = MIN(MIN(origin_iovec[origin_iov_index].iov_len, target_iovec[target_iov_index].iov_len), max_rdma_len); + + opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "performing swap on contiguous region. result: %p origin: %p, target: %p, len: %lu", + result_iovec[result_iov_index].iov_base, + origin_iovec[origin_iov_index].iov_base, + target_iovec[target_iov_index].iov_base, + (unsigned long) target_iovec[target_iov_index].iov_len)); + + ret = PtlFetchAtomic(result_md_h, + (ptl_size_t)result_iovec[result_iov_index].iov_base, + origin_md_h, + (ptl_size_t)origin_iovec[origin_iov_index].iov_base, + rdma_len, + peer, + pt_index, + match_bits, + offset + (ptl_size_t)target_iovec[target_iov_index].iov_base, + user_ptr, + 0, + ptl_op, + ptl_dt); + if (PTL_OK != ret) { + opal_output_verbose(1, ompi_osc_base_framework.framework_output, + "%s:%d PtlFetchAtomic failed with return value %d", + __FUNCTION__, __LINE__, ret); + opal_atomic_add_64(&module->opcount, -1); + return ret; + } + + /* adjust io vectors */ + result_iovec[result_iov_index].iov_len -= rdma_len; + origin_iovec[origin_iov_index].iov_len -= rdma_len; + target_iovec[target_iov_index].iov_len -= rdma_len; + result_iovec[result_iov_index].iov_base = (void *)((intptr_t) result_iovec[result_iov_index].iov_base + rdma_len); + origin_iovec[origin_iov_index].iov_base = (void *)((intptr_t) origin_iovec[origin_iov_index].iov_base + rdma_len); + target_iovec[target_iov_index].iov_base = (void *)((intptr_t) target_iovec[target_iov_index].iov_base + rdma_len); + + result_iov_index += (0 == result_iovec[result_iov_index].iov_len); + origin_iov_index += (0 == origin_iovec[origin_iov_index].iov_len); + target_iov_index += (0 == target_iovec[target_iov_index].iov_len); + } + } while (!done); + + return OMPI_SUCCESS; } int @@ -286,8 +2032,8 @@ ompi_osc_portals4_rput(const void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; - OPAL_PTRDIFF_TYPE length, origin_lb, target_lb; + size_t size, offset; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rput: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", @@ -302,39 +2048,76 @@ ompi_osc_portals4_rput(const void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rput: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + ret = put_to_noncontig(&module->opcount, + module->req_md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d put_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + ret = put_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d put_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } } else { - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - length *= origin_count; - request->ops_expected = number_of_fragment(length, mca_osc_portals4_component.ptl_max_msg_size); - opal_atomic_add_64(&module->opcount, request->ops_expected); - OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d Put", __FUNCTION__, __LINE__)); + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; - ret = splittedPtlPut(module->req_md_h, - (ptl_size_t) origin_addr + origin_lb, - length, - PTL_ACK_REQ, - peer, - module->pt_idx, - module->match_bits, - offset + target_lb, - request, - 0); + request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d RPut(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", + __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); + ret = segmentedPut(&module->opcount, + module->req_md_h, + (ptl_size_t) origin_addr + origin_lb, + length, + mca_osc_portals4_component.ptl_max_msg_size, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + request, + 0); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; @@ -361,8 +2144,8 @@ ompi_osc_portals4_rget(void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; - OPAL_PTRDIFF_TYPE length, origin_lb, target_lb; + size_t offset, size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", @@ -377,36 +2160,69 @@ ompi_osc_portals4_rget(void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rget: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + ret = get_from_noncontig(&module->opcount, + module->req_md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d get_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + ret = get_to_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d get_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } } else { - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - length *= origin_count; - request->ops_expected = number_of_fragment(length, mca_osc_portals4_component.ptl_max_msg_size); - opal_atomic_add_64(&module->opcount, request->ops_expected); + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; + + request->ops_expected += number_of_fragments(length, mca_osc_portals4_component.ptl_max_msg_size); + OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d Get", __FUNCTION__, __LINE__)); - ret = splittedPtlGet(module->req_md_h, - (ptl_size_t) origin_addr + origin_lb, - length, - peer, - module->pt_idx, - module->match_bits, - offset + target_lb, - request); + "%s,%d RGet", __FUNCTION__, __LINE__)); + ret = segmentedGet(&module->opcount, + module->req_md_h, + (ptl_size_t) origin_addr + origin_lb, + length, + mca_osc_portals4_component.ptl_max_msg_size, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + request); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; @@ -434,10 +2250,10 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; + size_t offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; - OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb; + OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "raccumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s 0x%lx", @@ -453,66 +2269,152 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Raccumulate: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + if (MPI_REPLACE == op) { + ret = atomic_put_to_noncontig(module, + module->req_md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_put_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + ret = atomic_to_noncontig(module, + module->req_md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + op, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + if (MPI_REPLACE == op) { + ret = atomic_put_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_put_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + ret = atomic_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + op, + request); + if (PTL_OK != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } } else { ptl_size_t md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - length *= origin_count; + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; sent = 0; md_offset = (ptl_size_t) origin_addr; - do { - size_t msg_length = MIN(module->atomic_max, length - sent); + request->ops_expected += number_of_fragments(length, module->atomic_max); - if (MPI_REPLACE == op) { - request->ops_expected += number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size); - opal_atomic_add_64(&module->opcount, number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size)); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Put", __FUNCTION__, __LINE__)); - ret = splittedPtlPut(module->req_md_h, - md_offset + sent + origin_lb, - msg_length, - PTL_ACK_REQ, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - request, - 0); - } else { - request->ops_expected++; - opal_atomic_add_64(&module->opcount, 1); - ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); - if (OMPI_SUCCESS != ret) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Raccumulate: datatype is not currently supported"); - return OMPI_ERR_NOT_SUPPORTED; - } + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Put", __FUNCTION__, __LINE__)); + ret = segmentedPut(&module->opcount, + module->req_md_h, + md_offset + origin_lb, + length, + module->atomic_max, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + request, + 0); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } else { + ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + do { + size_t msg_length = MIN(module->atomic_max, length - sent); - ret = ompi_osc_portals4_get_op(op, &ptl_op); - if (OMPI_SUCCESS != ret) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Raccumulate: operation is not currently supported"); - return OMPI_ERR_NOT_SUPPORTED; - } - OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, - "%s,%d Atomic", __FUNCTION__, __LINE__)); + (void)opal_atomic_add_64(&module->opcount, 1); + + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Atomic", __FUNCTION__, __LINE__)); ret = PtlAtomic(module->req_md_h, md_offset + sent + origin_lb, msg_length, @@ -525,13 +2427,14 @@ ompi_osc_portals4_raccumulate(const void *origin_addr, 0, ptl_op, ptl_dt); - } - if (OMPI_SUCCESS != ret) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - return ret; - } - sent += msg_length; - } while (sent < length); + if (OMPI_SUCCESS != ret) { + (void)opal_atomic_add_64(&module->opcount, -1); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + sent += msg_length; + } while (sent < length); + } } return OMPI_SUCCESS; @@ -558,10 +2461,10 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; + size_t target_offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; - OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb, result_lb; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, result_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "rget_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", @@ -577,126 +2480,272 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, if (NULL == request) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; *ompi_req = &request->super; - offset = get_displacement(module, target) * target_disp; + target_offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rget_accumulate: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (target_count > 0 && !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_REPLACE non-contiguous target")); + ret = swap_from_noncontig(module, + module->req_md_h, + result_addr, + result_count, + result_dt, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d swap_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_NO_OP non-contiguous target")); + ret = atomic_get_from_noncontig(module, + module->req_md_h, + result_addr, + result_count, + result_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_get_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: other-op non-contiguous target")); + ret = fetch_atomic_from_noncontig(module, + module->req_md_h, + result_addr, + result_count, + result_dt, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + op, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } + } else if ((origin_count > 0 && !ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) || + (result_count > 0 && !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count))) { + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_REPLACE non-contiguous origin/result")); + ret = swap_to_iovec(module, + result_addr, + result_count, + result_dt, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d swap_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_NO_OP non-contiguous origin/result")); + ret = atomic_get_to_iovec(module, + result_addr, + result_count, + result_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_get_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: other-op non-contiguous origin/result")); + ret = fetch_atomic_to_iovec(module, + result_addr, + result_count, + result_dt, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + op, + request); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d fetch_atomic_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } + } } else { - sent = 0; - if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_REPLACE contiguous")); ptl_size_t result_md_offset, origin_md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rget_accumulate: datatype is not currently supported"); + "datatype is not currently supported"); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } - length *= origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - (void)opal_atomic_add_64(&module->opcount, 1); - request->ops_expected++; + request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Swap", __FUNCTION__, __LINE__)); - ret = PtlSwap(module->req_md_h, - result_md_offset + sent + result_lb, - module->md_h, - origin_md_offset + sent + origin_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - request, - 0, - NULL, - PTL_SWAP, - ptl_dt); - sent += msg_length; - } while (sent < length); + ret = segmentedSwap(&module->opcount, + module->req_md_h, + result_md_offset + result_lb, + module->md_h, + origin_md_offset + origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + request, + ptl_dt); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: MPI_NO_OP contiguous")); ptl_size_t md_offset; - ret = ompi_datatype_get_extent(target_dt, &target_lb, &length); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - length *= target_count; + ompi_datatype_type_size(target_dt, &size); + length = size * target_count; md_offset = (ptl_size_t) result_addr; - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - opal_atomic_add_64(&module->opcount, number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size)); - request->ops_expected += number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Get", __FUNCTION__, __LINE__)); - ret = splittedPtlGet(module->req_md_h, - md_offset + sent + result_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - request); - sent += msg_length; - } while (sent < length); + request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); + + OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, + "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__)); + ret = segmentedGet(&module->opcount, + module->req_md_h, + (ptl_size_t) md_offset + result_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + request); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate: other-op contiguous")); ptl_size_t result_md_offset, origin_md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return ret; } - length *= origin_count; + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; @@ -704,44 +2753,39 @@ ompi_osc_portals4_rget_accumulate(const void *origin_addr, ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rget_accumulate: datatype is not currently supported"); + "datatype is not currently supported"); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } ret = ompi_osc_portals4_get_op(op, &ptl_op); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, - "MPI_Rget_accumulate: operation is not currently supported"); + "operation is not currently supported"); + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); return OMPI_ERR_NOT_SUPPORTED; } - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - (void)opal_atomic_add_64(&module->opcount, 1); - request->ops_expected++; - - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d FetchAtomic", __FUNCTION__, __LINE__)); - ret = PtlFetchAtomic(module->req_md_h, - result_md_offset + sent + result_lb, - module->md_h, - origin_md_offset + sent + origin_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - request, - 0, - ptl_op, - ptl_dt); - sent += msg_length; - } while (sent < length); - } - if (OMPI_SUCCESS != ret) { - OMPI_OSC_PORTALS4_REQUEST_RETURN(request); - return ret; + request->ops_expected += number_of_fragments(length, module->fetch_atomic_max); + + ret = segmentedFetchAtomic(&module->opcount, + module->req_md_h, + result_md_offset + result_lb, + module->md_h, + origin_md_offset + origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + request, + ptl_op, + ptl_dt); + if (OMPI_SUCCESS != ret) { + OMPI_OSC_PORTALS4_REQUEST_RETURN(request); + return ret; + } } } @@ -775,11 +2819,43 @@ ompi_osc_portals4_put(const void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Put: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + ret = put_to_noncontig(&module->opcount, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d put_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + ret = put_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d put_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } } else { ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { @@ -792,21 +2868,21 @@ ompi_osc_portals4_put(const void *origin_addr, ompi_datatype_type_size(origin_dt, &size); length = size * origin_count; - opal_atomic_add_64(&module->opcount, number_of_fragment(length, mca_osc_portals4_component.ptl_max_msg_size)); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Put(origin_count=%d, origin_lb=%lu, target_count=%d, target_lb=%lu, length=%lu, op_count=%ld)", __FUNCTION__, __LINE__, origin_count, origin_lb, target_count, target_lb, length, module->opcount)); - ret = splittedPtlPut(module->md_h, - (ptl_size_t) origin_addr + origin_lb, - length, - PTL_ACK_REQ, - peer, - module->pt_idx, - module->match_bits, - offset + target_lb, - NULL, - 0); + ret = segmentedPut(&module->opcount, + module->md_h, + (ptl_size_t) origin_addr + origin_lb, + length, + mca_osc_portals4_component.ptl_max_msg_size, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + NULL, + 0); if (OMPI_SUCCESS != ret) { return ret; } @@ -830,8 +2906,8 @@ ompi_osc_portals4_get(void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; - OPAL_PTRDIFF_TYPE length, origin_lb, target_lb; + size_t offset, size; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get: 0x%lx, %d, %s, %d, %lu, %d, %s, 0x%lx", @@ -842,32 +2918,67 @@ ompi_osc_portals4_get(void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Get: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + ret = get_from_noncontig(&module->opcount, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d get_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + ret = get_to_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d get_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } } else { - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - length *= origin_count; - opal_atomic_add_64(&module->opcount, number_of_fragment(length, mca_osc_portals4_component.ptl_max_msg_size)); + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; + OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, "%s,%d Get", __FUNCTION__, __LINE__)); - ret = splittedPtlGet(module->md_h, - (ptl_size_t) origin_addr + origin_lb, - length, - peer, - module->pt_idx, - module->match_bits, - offset + target_lb, - NULL); + ret = segmentedGet(&module->opcount, + module->md_h, + (ptl_size_t) origin_addr + origin_lb, + length, + mca_osc_portals4_component.ptl_max_msg_size, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + NULL); if (OMPI_SUCCESS != ret) { return ret; } @@ -892,10 +3003,10 @@ ompi_osc_portals4_accumulate(const void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; + size_t offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; - OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb; + OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "accumulate: 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", @@ -907,59 +3018,138 @@ ompi_osc_portals4_accumulate(const void *origin_addr, offset = get_displacement(module, target) * target_disp; - if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count) || - !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Accumulate: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (!ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + if (MPI_REPLACE == op) { + ret = atomic_put_to_noncontig(module, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_put_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + ret = atomic_to_noncontig(module, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + op, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_to_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } + } else if (!ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) { + if (MPI_REPLACE == op) { + ret = atomic_put_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_put_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + ret = atomic_from_iovec(module, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + offset, + module->pt_idx, + module->match_bits, + op, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_from_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } } else { ptl_size_t md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - length *= origin_count; + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; sent = 0; md_offset = (ptl_size_t) origin_addr; - do { - size_t msg_length = MIN(module->atomic_max, length - sent); + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d Put", __FUNCTION__, __LINE__)); + ret = segmentedPut(&module->opcount, + module->md_h, + md_offset + origin_lb, + length, + module->atomic_max, + PTL_ACK_REQ, + peer, + module->pt_idx, + module->match_bits, + offset + target_lb, + NULL, + 0); + if (OMPI_SUCCESS != ret) { + return ret; + } + } else { + ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "datatype is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + ret = ompi_osc_portals4_get_op(op, &ptl_op); + if (OMPI_SUCCESS != ret) { + opal_output(ompi_osc_base_framework.framework_output, + "operation is not currently supported"); + return OMPI_ERR_NOT_SUPPORTED; + } + do { + size_t msg_length = MIN(module->atomic_max, length - sent); - if (MPI_REPLACE == op) { - opal_atomic_add_64(&module->opcount, number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size)); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Put", __FUNCTION__, __LINE__)); - ret = splittedPtlPut(module->md_h, - md_offset + sent + origin_lb, - msg_length, - PTL_ACK_REQ, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - NULL, - 0); - } else { (void)opal_atomic_add_64(&module->opcount, 1); - ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); - if (OMPI_SUCCESS != ret) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Accumulate: datatype is not currently supported"); - return OMPI_ERR_NOT_SUPPORTED; - } - - ret = ompi_osc_portals4_get_op(op, &ptl_op); - if (OMPI_SUCCESS != ret) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Accumulate: operation is not currently supported"); - return OMPI_ERR_NOT_SUPPORTED; - } OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Atomic", __FUNCTION__, __LINE__)); @@ -975,12 +3165,13 @@ ompi_osc_portals4_accumulate(const void *origin_addr, 0, ptl_op, ptl_dt); - } - if (OMPI_SUCCESS != ret) { - return ret; - } - sent += msg_length; - } while (sent < length); + if (OMPI_SUCCESS != ret) { + (void)opal_atomic_add_64(&module->opcount, -1); + return ret; + } + sent += msg_length; + } while (sent < length); + } } return OMPI_SUCCESS; @@ -1005,10 +3196,10 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, ompi_osc_portals4_module_t *module = (ompi_osc_portals4_module_t*) win->w_osc_module; ptl_process_t peer = ompi_osc_portals4_get_peer(module, target); - size_t offset; + size_t target_offset, size; ptl_op_t ptl_op; ptl_datatype_t ptl_dt; - OPAL_PTRDIFF_TYPE sent, length, origin_lb, target_lb, result_lb; + OPAL_PTRDIFF_TYPE length, origin_lb, target_lb, result_lb, extent; OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "get_accumulate: 0x%lx, %d, %s, 0x%lx, %d, %s, %d, %lu, %d, %s, %s, 0x%lx", @@ -1020,113 +3211,251 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, op->o_name, (unsigned long) win)); - offset = get_displacement(module, target) * target_disp; + target_offset = get_displacement(module, target) * target_disp; - /* we don't support non-contiguous buffers. but if the count is 0, we don't care if buffer is non-contiguous. */ - if ((origin_count > 0 && !ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) || - (result_count > 0 && !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count)) || - (target_count > 0 && !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count))) { - opal_output(ompi_osc_base_framework.framework_output, - "MPI_Get_accumulate: transfer of non-contiguous memory is not currently supported.\n"); - return OMPI_ERR_NOT_SUPPORTED; + if (target_count > 0 && !ompi_datatype_is_contiguous_memory_layout(target_dt, target_count)) { + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_REPLACE non-contiguous target")); + ret = swap_from_noncontig(module, + module->md_h, + result_addr, + result_count, + result_dt, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d swap_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_NO_OP non-contiguous target")); + ret = atomic_get_from_noncontig(module, + module->md_h, + result_addr, + result_count, + result_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_get_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: other-op non-contiguous target")); + ret = fetch_atomic_from_noncontig(module, + module->md_h, + result_addr, + result_count, + result_dt, + module->md_h, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + op, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d fetch_atomic_from_noncontig() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } + } else if ((origin_count > 0 && !ompi_datatype_is_contiguous_memory_layout(origin_dt, origin_count)) || + (result_count > 0 && !ompi_datatype_is_contiguous_memory_layout(result_dt, result_count))) { + if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_REPLACE non-contiguous origin/result")); + ret = swap_to_iovec(module, + result_addr, + result_count, + result_dt, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d swap_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_NO_OP non-contiguous origin/result")); + ret = atomic_get_to_iovec(module, + result_addr, + result_count, + result_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d atomic_get_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: other-op non-contiguous origin/result")); + ret = fetch_atomic_to_iovec(module, + result_addr, + result_count, + result_dt, + origin_addr, + origin_count, + origin_dt, + peer, + target_count, + target_dt, + target_offset, + module->pt_idx, + module->match_bits, + op, + NULL); + if (PTL_OK != ret) { + OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, + "%s,%d fetch_atomic_to_iovec() failed: ret = %d", + __FUNCTION__, __LINE__, ret)); + return ret; + } + } } else { - sent = 0; if (MPI_REPLACE == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_REPLACE contiguous")); ptl_size_t result_md_offset, origin_md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; + ret = ompi_osc_portals4_get_dt(origin_dt, &ptl_dt); if (OMPI_SUCCESS != ret) { opal_output(ompi_osc_base_framework.framework_output, "MPI_Get_accumulate: datatype is not currently supported"); return OMPI_ERR_NOT_SUPPORTED; } - length *= origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - (void)opal_atomic_add_64(&module->opcount, 1); - - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Swap", __FUNCTION__, __LINE__)); - ret = PtlSwap(module->md_h, - result_md_offset + sent + result_lb, - module->md_h, - origin_md_offset + sent + origin_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - NULL, - 0, - NULL, - PTL_SWAP, - ptl_dt); - sent += msg_length; - } while (sent < length); + ret = segmentedSwap(&module->opcount, + module->md_h, + result_md_offset + result_lb, + module->md_h, + origin_md_offset + origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + NULL, + ptl_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } } else if (MPI_NO_OP == op) { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: MPI_NO_OP contiguous")); ptl_size_t md_offset; - ret = ompi_datatype_get_extent(target_dt, &target_lb, &length); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - length *= target_count; + ompi_datatype_type_size(target_dt, &size); + length = size * target_count; md_offset = (ptl_size_t) result_addr; - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - opal_atomic_add_64(&module->opcount, number_of_fragment(msg_length, mca_osc_portals4_component.ptl_max_msg_size)); - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d Get", __FUNCTION__, __LINE__)); - ret = splittedPtlGet(module->md_h, - md_offset + sent + result_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - NULL); - sent += msg_length; - } while (sent < length); + OPAL_OUTPUT_VERBOSE((90,ompi_osc_base_framework.framework_output, + "%s,%d MPI_Get_accumulate", __FUNCTION__, __LINE__)); + ret = segmentedGet(&module->opcount, + module->md_h, + (ptl_size_t) md_offset + result_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + NULL); + if (OMPI_SUCCESS != ret) { + return ret; + } } else { + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate: other-op contiguous")); ptl_size_t result_md_offset, origin_md_offset; - ret = ompi_datatype_get_extent(origin_dt, &origin_lb, &length); + ret = ompi_datatype_get_true_extent(origin_dt, &origin_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(target_dt, &target_lb); + ret = ompi_datatype_get_true_extent(target_dt, &target_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - ret = ompi_datatype_type_lb(result_dt, &result_lb); + ret = ompi_datatype_get_true_extent(result_dt, &result_lb, &extent); if (OMPI_SUCCESS != ret) { return ret; } - length *= origin_count; + ompi_datatype_type_size(origin_dt, &size); + length = size * origin_count; result_md_offset = (ptl_size_t) result_addr; origin_md_offset = (ptl_size_t) origin_addr; @@ -1145,31 +3474,23 @@ ompi_osc_portals4_get_accumulate(const void *origin_addr, return OMPI_ERR_NOT_SUPPORTED; } - do { - size_t msg_length = MIN(module->fetch_atomic_max, length - sent); - - (void)opal_atomic_add_64(&module->opcount, 1); - - OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, - "%s,%d FetchAtomic", __FUNCTION__, __LINE__)); - ret = PtlFetchAtomic(module->md_h, - result_md_offset + sent + result_lb, - module->md_h, - origin_md_offset + sent + origin_lb, - msg_length, - peer, - module->pt_idx, - module->match_bits, - offset + sent + target_lb, - NULL, - 0, - ptl_op, - ptl_dt); - sent += msg_length; - } while (sent < length); - } - if (OMPI_SUCCESS != ret) { - return ret; + ret = segmentedFetchAtomic(&module->opcount, + module->md_h, + result_md_offset + result_lb, + module->md_h, + origin_md_offset + origin_lb, + length, + module->fetch_atomic_max, + peer, + module->pt_idx, + module->match_bits, + target_offset + target_lb, + NULL, + ptl_op, + ptl_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } } } @@ -1314,10 +3635,10 @@ ompi_osc_portals4_fetch_and_op(const void *origin_addr, md_offset = (ptl_size_t) result_addr; - opal_atomic_add_64(&module->opcount, number_of_fragment(length, mca_osc_portals4_component.ptl_max_msg_size)); + (void)opal_atomic_add_64(&module->opcount, 1); OPAL_OUTPUT_VERBOSE((90, ompi_osc_base_framework.framework_output, "%s,%d Get", __FUNCTION__, __LINE__)); - ret = splittedPtlGet(module->md_h, + ret = PtlGet(module->md_h, md_offset, length, peer, diff --git a/ompi/mca/osc/portals4/osc_portals4_component.c b/ompi/mca/osc/portals4/osc_portals4_component.c index 889b20e8255..35c3b306458 100644 --- a/ompi/mca/osc/portals4/osc_portals4_component.c +++ b/ompi/mca/osc/portals4/osc_portals4_component.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2011-2017 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -508,6 +508,11 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit goto error; } + module->origin_iovec_list = NULL; + module->origin_iovec_md_h = PTL_INVALID_HANDLE; + module->result_iovec_list = NULL; + module->result_iovec_md_h = PTL_INVALID_HANDLE; + if (MPI_WIN_FLAVOR_DYNAMIC == flavor) { me.start = 0; me.length = PTL_SIZE_MAX; @@ -652,6 +657,14 @@ ompi_osc_portals4_free(struct ompi_win_t *win) PtlMEUnlink(module->control_me_h); PtlMEUnlink(module->data_me_h); PtlMDRelease(module->md_h); + if (module->origin_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->origin_iovec_md_h); + free(module->origin_iovec_list); + } + if (module->result_iovec_md_h != PTL_INVALID_HANDLE) { + PtlMDRelease(module->result_iovec_md_h); + free(module->result_iovec_list); + } PtlMDRelease(module->req_md_h); PtlCTFree(module->ct_h); if (NULL != module->disp_units) free(module->disp_units);