From 22401df03bec59300c10851587430328b68d05d8 Mon Sep 17 00:00:00 2001 From: George Bosilca Date: Tue, 9 May 2017 16:57:15 -0400 Subject: [PATCH 1/2] Allow MPI_ANY_SOURCE in MPI_Sendrecv_replace. Signed-off-by: George Bosilca (cherry picked from commit 86a7b317a5d9f228342cddf813dffad48e83a7d1) --- ompi/mpi/c/sendrecv_replace.c | 121 +++++++++++++++++----------------- 1 file changed, 60 insertions(+), 61 deletions(-) diff --git a/ompi/mpi/c/sendrecv_replace.c b/ompi/mpi/c/sendrecv_replace.c index 0063125119d..98b3089bfc2 100644 --- a/ompi/mpi/c/sendrecv_replace.c +++ b/ompi/mpi/c/sendrecv_replace.c @@ -2,7 +2,7 @@ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2010 The University of Tennessee and The University + * Copyright (c) 2004-2017 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, @@ -48,10 +48,10 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype, int rc = MPI_SUCCESS; MEMCHECKER( - memchecker_datatype(datatype); - memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype); - memchecker_comm(comm); - ); + memchecker_datatype(datatype); + memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype); + memchecker_comm(comm); + ); if ( MPI_PARAM_CHECK ) { rc = MPI_SUCCESS; @@ -76,68 +76,67 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype, /* simple case */ if ( source == MPI_PROC_NULL || dest == MPI_PROC_NULL || count == 0 ) { - rc = PMPI_Sendrecv(buf,count,datatype,dest,sendtag,buf,count,datatype,source,recvtag,comm,status); + rc = PMPI_Sendrecv(buf, count, datatype, dest, sendtag, buf, count, datatype, source, recvtag, comm, status); OPAL_CR_EXIT_LIBRARY(); return rc; - } else { - - opal_convertor_t convertor; - struct iovec iov; - unsigned char recv_data[2048]; - size_t packed_size, max_data; - uint32_t iov_count; - ompi_status_public_t recv_status; - ompi_proc_t* proc = ompi_comm_peer_lookup(comm,source); - if(proc == NULL) { - rc = MPI_ERR_RANK; - OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); - } - - /* initialize convertor to unpack recv buffer */ - OBJ_CONSTRUCT(&convertor, opal_convertor_t); - opal_convertor_copy_and_prepare_for_recv( proc->super.proc_convertor, &(datatype->super), - count, buf, 0, &convertor ); - - /* setup a buffer for recv */ - opal_convertor_get_packed_size( &convertor, &packed_size ); - if( packed_size > sizeof(recv_data) ) { - rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base); - if(OMPI_SUCCESS != rc) { - OMPI_ERRHANDLER_RETURN(OMPI_ERR_OUT_OF_RESOURCE, comm, MPI_ERR_BUFFER, FUNC_NAME); - } - } else { - iov.iov_base = (caddr_t)recv_data; - } - - /* recv into temporary buffer */ - rc = PMPI_Sendrecv( buf, count, datatype, dest, sendtag, iov.iov_base, packed_size, - MPI_BYTE, source, recvtag, comm, &recv_status ); - if (rc != MPI_SUCCESS) { - if(packed_size > sizeof(recv_data)) - PMPI_Free_mem(iov.iov_base); - OBJ_DESTRUCT(&convertor); - OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); - } - - /* unpack into users buffer */ - iov.iov_len = recv_status._ucount; - iov_count = 1; - max_data = recv_status._ucount; - opal_convertor_unpack(&convertor, &iov, &iov_count, &max_data ); + } - /* return status to user */ - if(status != MPI_STATUS_IGNORE) { - *status = recv_status; - } + /** + * If we look for an optimal solution, then we should receive the data into a temporary buffer + * and once the send completes we would unpack back into the original buffer. However, if the + * sender is unknown, this approach can only be implementing by receiving with the recv datatype + * (potentially non-contiguous) and thus the allocated memory will be larger than the size of the + * datatype. A simpler, but potentially less efficient approach is to work on the data we have + * control of, aka the sent data, and pack it into a contiguous buffer before posting the receive. + * Once the send completes, we free it. + */ + opal_convertor_t convertor; + unsigned char packed_data[2048]; + struct iovec iov = { .iov_base = packed_data, .iov_len = sizeof(packed_data) }; + size_t packed_size, max_data; + uint32_t iov_count; + ompi_status_public_t recv_status; + ompi_proc_t* proc = ompi_comm_peer_lookup(comm, dest); + if(proc == NULL) { + rc = MPI_ERR_RANK; + OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); + } - /* release resources */ - if(packed_size > sizeof(recv_data)) { - PMPI_Free_mem(iov.iov_base); + /* initialize convertor to unpack recv buffer */ + OBJ_CONSTRUCT(&convertor, opal_convertor_t); + opal_convertor_copy_and_prepare_for_send( proc->super.proc_convertor, &(datatype->super), + count, buf, 0, &convertor ); + + /* setup a buffer for recv */ + opal_convertor_get_packed_size( &convertor, &packed_size ); + if( packed_size > sizeof(packed_data) ) { + rc = PMPI_Alloc_mem(packed_size, MPI_INFO_NULL, &iov.iov_base); + if(OMPI_SUCCESS != rc) { + rc = OMPI_ERR_OUT_OF_RESOURCE; + goto cleanup_and_return; } - OBJ_DESTRUCT(&convertor); + } + max_data = packed_size; + iov_count = 1; + rc = opal_convertor_pack(&convertor, &iov, &iov_count, &max_data); + + /* recv into temporary buffer */ + rc = PMPI_Sendrecv( iov.iov_base, packed_size, MPI_PACKED, dest, sendtag, buf, count, + datatype, source, recvtag, comm, &recv_status ); + + cleanup_and_return: + /* return status to user */ + if(status != MPI_STATUS_IGNORE) { + *status = recv_status; + } - OPAL_CR_EXIT_LIBRARY(); - return MPI_SUCCESS; + /* release resources */ + if(packed_size > sizeof(packed_data)) { + PMPI_Free_mem(iov.iov_base); } + OBJ_DESTRUCT(&convertor); + + OPAL_CR_EXIT_LIBRARY(); + OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME); } From 9f412c3ce416cf3b40cccfa4f196d11914c90b41 Mon Sep 17 00:00:00 2001 From: William LePera Date: Thu, 1 Jun 2017 10:32:08 -0400 Subject: [PATCH 2/2] MPI_Sendreceive_replace data error with > 2k msg (RTC 155305) Signed-off-by: William LePera (cherry picked from commit a7c9c4aef35a7969cfe72542e5ed25ede66c0afd) --- ompi/mpi/c/sendrecv_replace.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ompi/mpi/c/sendrecv_replace.c b/ompi/mpi/c/sendrecv_replace.c index 98b3089bfc2..bb9f4126f13 100644 --- a/ompi/mpi/c/sendrecv_replace.c +++ b/ompi/mpi/c/sendrecv_replace.c @@ -12,6 +12,7 @@ * Copyright (c) 2010-2012 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -116,6 +117,7 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype, rc = OMPI_ERR_OUT_OF_RESOURCE; goto cleanup_and_return; } + iov.iov_len = packed_size; } max_data = packed_size; iov_count = 1;