From 519ddd9ae95a420880fcdde2c3fc5f9049bee04b Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 16:49:27 +0900 Subject: [PATCH 01/19] man: Insert missing error classes & Fix incorrect error codes. --- ompi/mpi/man/man3/MPI.3in | 200 +++++++++++++++++++++++--------------- 1 file changed, 122 insertions(+), 78 deletions(-) diff --git a/ompi/mpi/man/man3/MPI.3in b/ompi/mpi/man/man3/MPI.3in index 1d15de7f85e..9e99d67dc4d 100644 --- a/ompi/mpi/man/man3/MPI.3in +++ b/ompi/mpi/man/man3/MPI.3in @@ -66,85 +66,129 @@ For more information on Open MPI error codes, see \fImpi.h\fP in the \fIinclude\ Standard error return classes for Open MPI: .sp .nf -MPI_SUCCESS 0 Successful return code. -MPI_ERR_BUFFER 1 Invalid buffer pointer. -MPI_ERR_COUNT 2 Invalid count argument. -MPI_ERR_TYPE 3 Invalid datatype argument. -MPI_ERR_TAG 4 Invalid tag argument. -MPI_ERR_COMM 5 Invalid communicator. - -MPI_ERR_RANK 6 Invalid rank. -MPI_ERR_REQUEST 7 Invalid MPI_Request handle. -MPI_ERR_ROOT 7 Invalid root. -MPI_ERR_GROUP 8 Null group passed to function. -MPI_ERR_OP 9 Invalid operation. -MPI_ERR_TOPOLOGY 10 Invalid topology. - -MPI_ERR_DIMS 11 Illegal dimension argument. -MPI_ERR_ARG 12 Invalid argument. -MPI_ERR_UNKNOWN 13 Unknown error. -MPI_ERR_TRUNCATE 14 Message truncated on receive. -MPI_ERR_OTHER 15 Other error; use Error_string. - -MPI_ERR_INTERN 16 Internal error code. -MPI_ERR_IN_STATUS 17 Look in status for error value. -MPI_ERR_PENDING 18 Pending request. -MPI_ERR_ACCESS 19 Permission denied. -MPI_ERR_AMODE 20 Unsupported amode passed to open. - -MPI_ERR_ASSERT 21 Invalid assert. -MPI_ERR_BAD_FILE 22 Invalid file name (for example, - path name too long). -MPI_ERR_BASE 23 Invalid base. -MPI_ERR_CONVERSION 24 An error occurred in a user-supplied - data-conversion function. -MPI_ERR_DISP 25 Invalid displacement. - -MPI_ERR_DUP_DATAREP 26 Conversion functions could not - be registered because a data - representation identifier that was - already defined was passed to - MPI_REGISTER_DATAREP. -MPI_ERR_FILE_EXISTS 27 File exists. -MPI_ERR_FILE_IN_USE 28 File operation could not be - completed, as the file is currently - open by some process. -MPI_ERR_FILE 29 -MPI_ERR_INFO_KEY 30 Illegal info key. - -MPI_ERR_INFO_NOKEY 31 No such key. -MPI_ERR_INFO_VALUE 32 Illegal info value. -MPI_ERR_INFO 33 Invalid info object. -MPI_ERR_IO 34 I/O error. -MPI_ERR_KEYVAL 35 Illegal key value. - -MPI_ERR_LOCKTYPE 36 Invalid locktype. -MPI_ERR_NAME 37 Name not found. -MPI_ERR_NO_MEM 38 Memory exhausted. -MPI_ERR_NOT_SAME 39 -MPI_ERR_NO_SPACE 40 Not enough space. - -MPI_ERR_NO_SUCH_FILE 41 File (or directory) does not exist. -MPI_ERR_PORT 42 Invalid port. -MPI_ERR_QUOTA 43 Quota exceeded. -MPI_ERR_READ_ONLY 44 Read-only file system. -MPI_ERR_RMA_CONFLICT 45 Conflicting accesses to window. - -MPI_ERR_RMA_SYNC 46 Erroneous RMA synchronization. -MPI_ERR_SERVICE 47 Invalid publish/unpublish. -MPI_ERR_SIZE 48 Invalid size. -MPI_ERR_SPAWN 49 Error spawning. -MPI_ERR_UNSUPPORTED_DATAREP - 50 Unsupported datarep passed to - MPI_File_set_view. +MPI_SUCCESS 0 Successful return code. +MPI_ERR_BUFFER 1 Invalid buffer pointer. +MPI_ERR_COUNT 2 Invalid count argument. +MPI_ERR_TYPE 3 Invalid datatype argument. +MPI_ERR_TAG 4 Invalid tag argument. +MPI_ERR_COMM 5 Invalid communicator. + +MPI_ERR_RANK 6 Invalid rank. +MPI_ERR_REQUEST 7 Invalid MPI_Request handle. +MPI_ERR_ROOT 8 Invalid root. +MPI_ERR_GROUP 9 Null group passed to function. +MPI_ERR_OP 10 Invalid operation. + +MPI_ERR_TOPOLOGY 11 Invalid topology. +MPI_ERR_DIMS 12 Illegal dimension argument. +MPI_ERR_ARG 13 Invalid argument. +MPI_ERR_UNKNOWN 14 Unknown error. +MPI_ERR_TRUNCATE 15 Message truncated on receive. + +MPI_ERR_OTHER 16 Other error; use Error_string. +MPI_ERR_INTERN 17 Internal error code. +MPI_ERR_IN_STATUS 18 Look in status for error value. +MPI_ERR_PENDING 19 Pending request. +MPI_ERR_ACCESS 20 Permission denied. + +MPI_ERR_AMODE 21 Unsupported amode passed to open. +MPI_ERR_ASSERT 22 Invalid assert. +MPI_ERR_BAD_FILE 23 Invalid file name (for example, + path name too long). +MPI_ERR_BASE 24 Invalid base. +MPI_ERR_CONVERSION 25 An error occurred in a user-supplied + data-conversion function. + +MPI_ERR_DISP 26 Invalid displacement. +MPI_ERR_DUP_DATAREP 27 Conversion functions could not + be registered because a data + representation identifier that was + already defined was passed to + MPI_REGISTER_DATAREP. +MPI_ERR_FILE_EXISTS 28 File exists. +MPI_ERR_FILE_IN_USE 29 File operation could not be + completed, as the file is currently + open by some process. +MPI_ERR_FILE 30 Invalid file handle. + +MPI_ERR_INFO_KEY 31 Illegal info key. +MPI_ERR_INFO_NOKEY 32 No such key. +MPI_ERR_INFO_VALUE 33 Illegal info value. +MPI_ERR_INFO 34 Invalid info object. +MPI_ERR_IO 35 I/O error. + +MPI_ERR_KEYVAL 36 Illegal key value. +MPI_ERR_LOCKTYPE 37 Invalid locktype. +MPI_ERR_NAME 38 Name not found. +MPI_ERR_NO_MEM 39 Memory exhausted. +MPI_ERR_NOT_SAME 40 Collective argument not identical + on all processes, or collective + routines called in a different order + by different processes. + +MPI_ERR_NO_SPACE 41 Not enough space. +MPI_ERR_NO_SUCH_FILE 42 File (or directory) does not exist. +MPI_ERR_PORT 43 Invalid port. +MPI_ERR_QUOTA 44 Quota exceeded. +MPI_ERR_READ_ONLY 45 Read-only file system. + +MPI_ERR_RMA_CONFLICT 46 Conflicting accesses to window. +MPI_ERR_RMA_SYNC 47 Erroneous RMA synchronization. +MPI_ERR_SERVICE 48 Invalid publish/unpublish. +MPI_ERR_SIZE 49 Invalid size. +MPI_ERR_SPAWN 50 Error spawning. +MPI_ERR_UNSUPPORTED_DATAREP + 51 Unsupported datarep passed to + MPI_File_set_view. MPI_ERR_UNSUPPORTED_OPERATION - 51 Unsupported operation, such as - seeking on a file that supports - only sequential access. -MPI_ERR_WIN 52 Invalid window. -MPI_ERR_LASTCODE 53 Last error code. - -MPI_ERR_SYSRESOURCE -2 Out of resources + 52 Unsupported operation, such as + seeking on a file that supports + only sequential access. +MPI_ERR_WIN 53 Invalid window. +MPI_T_ERR_MEMORY 54 Out of memory. +MPI_T_ERR_NOT_INITIALIZED 55 Interface not initialized. + +MPI_T_ERR_CANNOT_INIT 56 Interface not in the state to be + initialized. +MPI_T_ERR_INVALID_INDEX 57 The enumeration index is invalid. +MPI_T_ERR_INVALID_ITEM 58 The item index queried is out of + range. +MPI_T_ERR_INVALID_HANDLE 59 The handle is invalid. +MPI_T_ERR_OUT_OF_HANDLES 60 No more handles available. + +MPI_T_ERR_OUT_OF_SESSIONS 61 No more sessions available. +MPI_T_ERR_INVALID_SESSION 62 Session argument is not a valid + session. +MPI_T_ERR_CVAR_SET_NOT_NOW + 63 Variable cannot be set at this + moment. +MPI_T_ERR_CVAR_SET_NEVER 64 Variable cannot be set until end of + execution. +MPI_T_ERR_PVAR_NO_STARTSTOP + 65 Variable cannot be started or stopped. + +MPI_T_ERR_PVAR_NO_WRITE 66 Variable cannot be written or reset. +MPI_T_ERR_PVAR_NO_ATOMIC 67 Variable cannot be read and written + atomically. +MPI_ERR_RMA_RANGE 68 Target memory is not part of the + window (in the case of a window + created with MPI_WIN_CREATE_DYNAMIC, + target memory is not attached). +MPI_ERR_RMA_ATTACH 69 Memory cannot be attached (e.g., + because of resource exhaustion). +MPI_ERR_RMA_FLAVOR 70 Passed window has the wrong flavor + for the called function. + +MPI_ERR_RMA_SHARED 71 Memory cannot be shared (e.g., some + process in the group of the specified + communicator cannot expose shared + memory). +MPI_T_ERR_INVALID 72 Invalid use of the interface or bad + parameter values(s). +MPI_T_ERR_INVALID_NAME 73 The variable or category name is + invalid. + +MPI_ERR_LASTCODE 93 Last error code. .fi From 4bbe86b171b7b63cf3a3f915d0dd6ddbb7180835 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:14:22 +0900 Subject: [PATCH 02/19] man: Fix a typo of an argument. --- ompi/mpi/man/man3/MPI_Compare_and_swap.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Compare_and_swap.3in b/ompi/mpi/man/man3/MPI_Compare_and_swap.3in index 830445a92d6..0a28b12420a 100644 --- a/ompi/mpi/man/man3/MPI_Compare_and_swap.3in +++ b/ompi/mpi/man/man3/MPI_Compare_and_swap.3in @@ -13,7 +13,7 @@ .SH C Syntax .nf #include -int MPI_Compare_and_swap(const void *\fIorigin_addr\fP, const void *\fIcompar_addr\fP, +int MPI_Compare_and_swap(const void *\fIorigin_addr\fP, const void *\fIcompare_addr\fP, void *\fIresult_addr\fP, MPI_Datatype \fIdatatype\fP, int \fItarget_rank\fP, MPI_Aint \fItarget_disp\fP, MPI_Win \fIwin\fP) From 80a0b30be818a3c792d9259a0224e4ee527c89ad Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:15:30 +0900 Subject: [PATCH 03/19] man: Correct wrong argument order. --- ompi/mpi/man/man3/MPI_Alltoallv.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Alltoallv.3in b/ompi/mpi/man/man3/MPI_Alltoallv.3in index 02b9b8ad9eb..79fed316094 100644 --- a/ompi/mpi/man/man3/MPI_Alltoallv.3in +++ b/ompi/mpi/man/man3/MPI_Alltoallv.3in @@ -40,7 +40,7 @@ MPI_ALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE, INTEGER \fICOMM, IERROR\fP MPI_IALLTOALLV(\fISENDBUF, SENDCOUNTS, SDISPLS, SENDTYPE, - RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, REQUEST, COMM, IERROR\fP) + RECVBUF, RECVCOUNTS, RDISPLS, RECVTYPE, COMM, REQUEST, IERROR\fP) \fISENDBUF(*), RECVBUF(*)\fP INTEGER \fISENDCOUNTS(*), SDISPLS(*), SENDTYPE\fP From ffce87328d769b4518e1690003c11d17c8d8e9d3 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:17:45 +0900 Subject: [PATCH 04/19] man: MPI_Get_version now returns 3.1 instead of 2.1. --- ompi/mpi/man/man3/MPI_Get_version.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Get_version.3in b/ompi/mpi/man/man3/MPI_Get_version.3in index ec06521e499..56ef789c8a2 100644 --- a/ompi/mpi/man/man3/MPI_Get_version.3in +++ b/ompi/mpi/man/man3/MPI_Get_version.3in @@ -55,7 +55,7 @@ Fortran only: Error status (integer). .SH DESCRIPTION .ft R -Since Open MPI is MPI 2.1 compliant, this function will return a version value of 2 and a subversion value of 1 for this release. +Since Open MPI is MPI 3.1 compliant, this function will return a version value of 3 and a subversion value of 1 for this release. .SH NOTE .ft R From 9b96209ac52a8f33a341d14d6ab7e0de2453b683 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:34:28 +0900 Subject: [PATCH 05/19] man: Fix incorrect C++ binding descriptions. --- ompi/mpi/man/man3/MPI_Barrier.3in | 4 ++-- ompi/mpi/man/man3/MPI_Bsend_init.3in | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_Barrier.3in b/ompi/mpi/man/man3/MPI_Barrier.3in index edf34dd042c..3a18c2abe71 100644 --- a/ompi/mpi/man/man3/MPI_Barrier.3in +++ b/ompi/mpi/man/man3/MPI_Barrier.3in @@ -43,8 +43,8 @@ MPI_Ibarrier(\fIcomm\fP, \fIrequest\fP, \fIierror\fP) .fi .SH C++ Syntax .nf -import mpi.*; -void MPI.COMM_WORLD.Barrier() +#include +void MPI::Comm::Barrier() const = 0 .fi .SH INPUT PARAMETER diff --git a/ompi/mpi/man/man3/MPI_Bsend_init.3in b/ompi/mpi/man/man3/MPI_Bsend_init.3in index c1499f26a83..34309b31db3 100644 --- a/ompi/mpi/man/man3/MPI_Bsend_init.3in +++ b/ompi/mpi/man/man3/MPI_Bsend_init.3in @@ -42,8 +42,8 @@ MPI_Bsend_init(\fIbuf\fP, \fIcount\fP, \fIdatatype\fP, \fIdest\fP, \fItag\fP, \f .fi .SH C++ Syntax .nf -import mpi.*; -Prequest MPI.COMM_WORLD.Bsend_init(const void* \fIbuf\fP, int \fIcount\fP, const +#include +Prequest Comm::Bsend_init(const void* \fIbuf\fP, int \fIcount\fP, const Datatype& \fIdatatype\fP, int \fIdest\fP, int \fItag\fP) const .fi From 1261b115e43549cfc625af921b92f91c625a0e30 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:44:53 +0900 Subject: [PATCH 06/19] man: Fix incorrect nroff markups. --- ompi/mpi/man/man3/MPI_Put.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Put.3in b/ompi/mpi/man/man3/MPI_Put.3in index 64ae0f78618..17d59186974 100644 --- a/ompi/mpi/man/man3/MPI_Put.3in +++ b/ompi/mpi/man/man3/MPI_Put.3in @@ -123,7 +123,7 @@ The communication must satisfy the same constraints as for a similar message-pas .sp The \fItarget_datatype\fP argument is a handle to a datatype object defined at the origin process. However, this object is interpreted at the target process: The outcome is as if the target datatype object were defined at the target process, by the same sequence of calls used to define it at the origin process. The target data type must contain only relative displacements, not absolute addresses. The same holds for get and accumulate. .sp -\fBMPI_Rput\bP is similar to \fBMPI_Put\fI, except that it allocates a communication request object and associates it with the request handle (the argument \fIrequest\fP). The completion of an MPI_Rput operation (i.e., after the corresponding test or wait) indicates that the sender is now free to update the locations in the \fIorigin_addr\fP buffer. It does not indicate that the data is available at the target window. If remote completion is required, \fBMPI_Win_flush\fP, \fBMPI_Win_flush_all\fP, \fBMPI_Win_unlock\fP, or \fBMPI_Win_unlock_all\fP can be used. +\fBMPI_Rput\fP is similar to \fBMPI_Put\fP, except that it allocates a communication request object and associates it with the request handle (the argument \fIrequest\fP). The completion of an MPI_Rput operation (i.e., after the corresponding test or wait) indicates that the sender is now free to update the locations in the \fIorigin_addr\fP buffer. It does not indicate that the data is available at the target window. If remote completion is required, \fBMPI_Win_flush\fP, \fBMPI_Win_flush_all\fP, \fBMPI_Win_unlock\fP, or \fBMPI_Win_unlock_all\fP can be used. .SH NOTES The \fItarget_datatype\fP argument is a handle to a datatype object that is defined at the origin process, even though it defines a data layout in the target process memory. This does not cause problems in a homogeneous or heterogeneous environment, as long as only portable data types are used (portable data types are defined in Section 2.4 of the MPI-2 Standard). From 5a3b8b34cd75dcd4bbe85e0ffdf1806e1e533fc6 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Thu, 15 Oct 2015 17:51:02 +0900 Subject: [PATCH 07/19] man: Remove outdated description. MPI-2.2 is ratified. --- ompi/mpi/man/man3/MPI_Reduce_local.3in | 2 -- 1 file changed, 2 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_Reduce_local.3in b/ompi/mpi/man/man3/MPI_Reduce_local.3in index 6bca5d1c20e..bc944137d01 100644 --- a/ompi/mpi/man/man3/MPI_Reduce_local.3in +++ b/ompi/mpi/man/man3/MPI_Reduce_local.3in @@ -72,8 +72,6 @@ Fortran only: Error status (integer). .SH DESCRIPTION .ft R -.I The MPI_Reduce_local function is proposed for MPI-2.2 and (as of 10 Jan 2009) has not yet been ratified. Use at your own risk. See https://svn.mpi-forum.org/trac/mpi-forum-web/ticket/24. -.sp The global reduce functions (MPI_Reduce_local, MPI_Op_create, MPI_Op_free, MPI_Allreduce, MPI_Reduce_local_scatter, MPI_Scan) perform a global reduce operation (such as sum, max, logical AND, etc.) across all the members of a group. The reduction operation can be either one of a predefined list of operations, or a user-defined operation. The global reduction functions come in several flavors: a reduce that returns the result of the reduction at one node, an all-reduce that returns this result at all nodes, and a scan (parallel prefix) operation. In addition, a reduce-scatter operation combines the functionality of a reduce and a scatter operation. .sp MPI_Reduce_local combines the elements provided in the input and input/output buffers of the local process, using the operation op, and returns the combined value in the inout/output buffer. The input buffer is defined by the arguments inbuf, count, and datatype; the output buffer is defined by the arguments inoutbuf, count, and datatype; both have the same number of elements, with the same type. The routine is a local call. The process can provide one element, or a sequence of elements, in which case the combine operation is executed element-wise on each entry of the sequence. For example, if the operation is MPI_MAX and the input buffer contains two elements that are floating-point numbers (count = 2 and datatype = MPI_FLOAT), then inoutbuf(1) = global max (inbuf(1)) and inoutbuf(2) = global max(inbuf(2)). From 953c95e9bb46074e27c469d8207fee6c242955db Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Fri, 16 Oct 2015 16:12:43 +0900 Subject: [PATCH 08/19] man: Update description of MPI_IN_PLACE of MPI_Exscan. MPI-2.2 added MPI_IN_PLACE support for MPI_Exscan. --- ompi/mpi/man/man3/MPI_Exscan.3in | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Exscan.3in b/ompi/mpi/man/man3/MPI_Exscan.3in index 5da0f0a7033..fe9a3a14431 100644 --- a/ompi/mpi/man/man3/MPI_Exscan.3in +++ b/ompi/mpi/man/man3/MPI_Exscan.3in @@ -114,7 +114,13 @@ as \fIrecvbuf\fP is not significant for process 0. The value of \fIrecvbuf\fP on process 1 is always the value in \fIsendbuf\fP on process 0. .sp -No MPI_IN_PLACE operation is supported. +.SH USE OF IN-PLACE OPTION +The `in place' option for intracommunicators is specified by passing MPI_IN_PLACE in the \fIsendbuf\fP argument. In this case, the input data is taken from the receive buffer, and replaced by the output data. +.sp +Note that MPI_IN_PLACE is a special kind of value; it has the same restrictions on its use as MPI_BOTTOM. +.sp +Because the in-place option converts the receive buffer into a send-and-receive buffer, a Fortran binding that includes INTENT must mark these as INOUT, not OUT. +.sp .SH NOTES .ft R From 9942d5a93383173d85a19c928d5b0c08d44cc70d Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Fri, 16 Oct 2015 16:48:43 +0900 Subject: [PATCH 09/19] man: MPI_IBARRIER has two output parameters. --- ompi/mpi/man/man3/MPI_Barrier.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Barrier.3in b/ompi/mpi/man/man3/MPI_Barrier.3in index 3a18c2abe71..9e84efed9db 100644 --- a/ompi/mpi/man/man3/MPI_Barrier.3in +++ b/ompi/mpi/man/man3/MPI_Barrier.3in @@ -53,7 +53,7 @@ void MPI::Comm::Barrier() const = 0 comm Communicator (handle). -.SH OUTPUT PARAMETER +.SH OUTPUT PARAMETERS .ft R .TP 1i request From bffc7b6c8f353bc0c7f3cae47433866319fa00e9 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 13:43:36 +0900 Subject: [PATCH 10/19] man: Add man of `MPI_Message_{c2f,f2c}` and `MPI_Op_commutative`. These routines were added in MPI-2.2 but were missing in OMPI man pages. --- ompi/mpi/man/man3/MPI_Comm_f2c.3in | 5 +- ompi/mpi/man/man3/MPI_Message_c2f.3in | 1 + ompi/mpi/man/man3/MPI_Message_f2c.3in | 1 + ompi/mpi/man/man3/MPI_Op_commutative.3in | 68 ++++++++++++++++++++++++ ompi/mpi/man/man3/Makefile.extra | 3 ++ 5 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 ompi/mpi/man/man3/MPI_Message_c2f.3in create mode 100644 ompi/mpi/man/man3/MPI_Message_f2c.3in create mode 100644 ompi/mpi/man/man3/MPI_Op_commutative.3in diff --git a/ompi/mpi/man/man3/MPI_Comm_f2c.3in b/ompi/mpi/man/man3/MPI_Comm_f2c.3in index ffb2b177fc7..734ee978097 100644 --- a/ompi/mpi/man/man3/MPI_Comm_f2c.3in +++ b/ompi/mpi/man/man3/MPI_Comm_f2c.3in @@ -4,7 +4,7 @@ .\" $COPYRIGHT$ .TH MPI_Comm_f2c 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" .SH NAME -\fBMPI_Comm_f2c, MPI_Comm_c2f, MPI_File_f2c, MPI_File_c2f, MPI_Info_f2c, MPI_Info_c2f, MPI_Op_f2c, MPI_Op_c2f, MPI_Request_f2c, MPI_Request_c2f, MPI_Type_f2c, MPI_Type_c2f, MPI_Win_f2c, MPI_Win_c2f \fP \- Translates a C handle into a Fortran handle, or vice versa. +\fBMPI_Comm_f2c, MPI_Comm_c2f, MPI_File_f2c, MPI_File_c2f, MPI_Info_f2c, MPI_Info_c2f, MPI_Message_f2c, MPI_Message_c2f, MPI_Op_f2c, MPI_Op_c2f, MPI_Request_f2c, MPI_Request_c2f, MPI_Type_f2c, MPI_Type_c2f, MPI_Win_f2c, MPI_Win_c2f \fP \- Translates a C handle into a Fortran handle, or vice versa. .SH SYNTAX .ft R @@ -23,6 +23,9 @@ MPI_Fint MPI_Group_c2f(MPI Group \fIgroup\fP) MPI_Info MPI_Info_f2c(MPI_Fint \fIinfo\fP) MPI_Fint MPI_Info_c2f(MPI_Info \fIinfo\fP) +MPI_Message MPI_Message_f2c(MPI_Fint \fImessage\fP) +MPI_Fint MPI_Message_c2f(MPI_Message \fImessage\fP) + MPI_Op MPI_Op_f2c(MPI_Fint \fIop\fP) MPI_Fint MPI_Op_c2f(MPI_Op \fIop\fP) diff --git a/ompi/mpi/man/man3/MPI_Message_c2f.3in b/ompi/mpi/man/man3/MPI_Message_c2f.3in new file mode 100644 index 00000000000..a13fce697dd --- /dev/null +++ b/ompi/mpi/man/man3/MPI_Message_c2f.3in @@ -0,0 +1 @@ +.so man3/MPI_Comm_f2c.3 diff --git a/ompi/mpi/man/man3/MPI_Message_f2c.3in b/ompi/mpi/man/man3/MPI_Message_f2c.3in new file mode 100644 index 00000000000..a13fce697dd --- /dev/null +++ b/ompi/mpi/man/man3/MPI_Message_f2c.3in @@ -0,0 +1 @@ +.so man3/MPI_Comm_f2c.3 diff --git a/ompi/mpi/man/man3/MPI_Op_commutative.3in b/ompi/mpi/man/man3/MPI_Op_commutative.3in new file mode 100644 index 00000000000..536cb9a7adb --- /dev/null +++ b/ompi/mpi/man/man3/MPI_Op_commutative.3in @@ -0,0 +1,68 @@ +.\" -*- nroff -*- +.\" Copyright 2015 FUJITSU LIMITED. All rights reserved. +.\" $COPYRIGHT$ +.TH MPI_Op_commutative 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#" +.SH NAME +\fBMPI_Op_commutative\fP \- Query of commutativity of reduction operation. + +.SH SYNTAX +.ft R +.SH C Syntax +.nf +#include +int MPI_Op_commutative(MPI_Op \fIop\fP, int *\fIcommute\fP) + +.fi +.SH Fortran Syntax +.nf +USE MPI +! or the older form: INCLUDE 'mpif.h' +MPI_OP_COMMUTATIVE(\fIOP, COMMUTE, IERROR\fP) + LOGICAL \fICOMMUTE\fP + INTEGER \fIOP, IERROR\fP + +.fi +.SH Fortran 2008 Syntax +.nf +USE mpi_f08 +MPI_Op_commutative(\fIop\fP, \fIcommute\fP, \fIierror\fP) + TYPE(MPI_Op), INTENT(IN) :: \fIop\fP + INTEGER, INTENT(OUT) :: \fIcommute\fP + INTEGER, OPTIONAL, INTENT(OUT) :: \fIierror\fP + +.fi +.SH C++ Syntax +.nf +#include +bool Op::Is_commutative() + +.fi +.SH INPUT PARAMETER +.TP 1i +op +Operation (handle). + +.SH OUTPUT PARAMETERS +.ft R +.TP 1i +commute +True if op is commutative, false otherwise (logical). +.ft R +.TP 1i +IERROR +Fortran only: Error status (integer). + +.SH DESCRIPTION +.ft R +Reduction operations can be queried for their commutativity. + +.SH ERRORS +Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object. +.sp +Before the error value is returned, the current MPI error handler is +called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error. + +.SH SEE ALSO +.sp +MPI_Op_create + diff --git a/ompi/mpi/man/man3/Makefile.extra b/ompi/mpi/man/man3/Makefile.extra index c4484ce25e9..4fe77c71907 100644 --- a/ompi/mpi/man/man3/Makefile.extra +++ b/ompi/mpi/man/man3/Makefile.extra @@ -235,6 +235,8 @@ mpi_api_man_pages = \ mpi/man/man3/MPI_Keyval_create.3 \ mpi/man/man3/MPI_Keyval_free.3 \ mpi/man/man3/MPI_Lookup_name.3 \ + mpi/man/man3/MPI_Message_c2f.3 \ + mpi/man/man3/MPI_Message_f2c.3 \ mpi/man/man3/MPI_Mprobe.3 \ mpi/man/man3/MPI_Mrecv.3 \ mpi/man/man3/MPI_Neighbor_allgather.3 \ @@ -248,6 +250,7 @@ mpi_api_man_pages = \ mpi/man/man3/MPI_Neighbor_alltoallw.3 \ mpi/man/man3/MPI_Ineighbor_alltoallw.3 \ mpi/man/man3/MPI_Op_c2f.3 \ + mpi/man/man3/MPI_Op_commutative.3 \ mpi/man/man3/MPI_Op_create.3 \ mpi/man/man3/MPI_Open_port.3 \ mpi/man/man3/MPI_Op_f2c.3 \ From 2226cdb3dacd21dcf7e9836db57ddb9d7936ac43 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 14:44:49 +0900 Subject: [PATCH 11/19] man: Correct the routine name of `MPI_FILE_IWRITE_SHARED`. --- ompi/mpi/man/man3/MPI_File_iwrite_shared.3in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in index 8391a777eae..58e3c539424 100644 --- a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in +++ b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in @@ -15,14 +15,14 @@ .nf C Syntax #include - int MPI_File_(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype - \fIdatatype\fP, MPI_Request \fI*request\fP) + int MPI_File_iwrite_shared(MPI_File \fIfh\fP, void \fI*buf\fP, int \fIcount\fP, + MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP) .fi .SH Fortran Syntax .nf INCLUDE 'mpif.h' - MPI_File_(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP,\fI IERROR\fP) + MPI_FILE_IWRITE_SHARED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP,\fI IERROR\fP) BUF(*) INTEGER FH, COUNT, DATATYPE, REQUEST, IERROR From 3253a30ab2922fa9eef2e0b5e7843a949a650853 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 14:48:23 +0900 Subject: [PATCH 12/19] man: Remove unnecessary spaces in front of parameters. --- ompi/mpi/man/man3/MPI_File_read.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_File_read.3in b/ompi/mpi/man/man3/MPI_File_read.3in index 7c19ee60350..b46c7fee781 100644 --- a/ompi/mpi/man/man3/MPI_File_read.3in +++ b/ompi/mpi/man/man3/MPI_File_read.3in @@ -19,7 +19,7 @@ C Syntax .SH Fortran Syntax .nf INCLUDE 'mpif.h' - MPI_FILE_READ(\fI FH\fP, \fI BUF\fP, \fICOUNT\fP, + MPI_FILE_READ(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fI DATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP) BUF(*) INTEGER FH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE), From 34c3b5d74d764a24d5f29240138f8349cebe8286 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 15:01:30 +0900 Subject: [PATCH 13/19] man: Correct the kind of `ADDRESS` parameter of `MPI_GET_ADDRESS`. --- ompi/mpi/man/man3/MPI_Get_address.3in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Get_address.3in b/ompi/mpi/man/man3/MPI_Get_address.3in index 8f643c5ac1c..5ed0b5e470f 100644 --- a/ompi/mpi/man/man3/MPI_Get_address.3in +++ b/ompi/mpi/man/man3/MPI_Get_address.3in @@ -22,7 +22,8 @@ USE MPI ! or the older form: INCLUDE 'mpif.h' MPI_GET_ADDRESS(\fILOCATION, ADDRESS, IERROR\fP) \fILOCATION\fP(*) - INTEGER \fIADDRESS, IERROR\fP + INTEGER(KIND=MPI_ADDRESS_KIND) \fIADDRESS\fP + INTEGER \fIIERROR\fP .fi .SH Fortran 2008 Syntax From dcd14103d5d263733c64c27e1842a6954ba8169b Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 15:04:50 +0900 Subject: [PATCH 14/19] man: Remove unnecessary spaces in Fortran syntax. Similar lines of other routines have no space. --- ompi/mpi/man/man3/MPI_Grequest_start.3in | 2 +- ompi/mpi/man/man3/MPI_Pack_external.3in | 2 +- ompi/mpi/man/man3/MPI_Pack_external_size.3in | 2 +- ompi/mpi/man/man3/MPI_Unpack_external.3in | 2 +- ompi/mpi/man/man3/MPI_Win_shared_query.3in | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_Grequest_start.3in b/ompi/mpi/man/man3/MPI_Grequest_start.3in index f96d7326809..e2738508882 100644 --- a/ompi/mpi/man/man3/MPI_Grequest_start.3in +++ b/ompi/mpi/man/man3/MPI_Grequest_start.3in @@ -26,7 +26,7 @@ MPI_GREQUEST_START(\fIQUERY_FN, FREE_FN, CANCEL_FN, EXTRA_STATE, REQUEST, IERROR\fP) INTEGER \fIREQUEST, IERROR\fP EXTERNAL \fIQUERY_FN, FREE_FN, CANCEL_FN\fP - INTEGER (KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE\fP + INTEGER(KIND=MPI_ADDRESS_KIND) \fIEXTRA_STATE\fP .fi .SH Fortran 2008 Syntax diff --git a/ompi/mpi/man/man3/MPI_Pack_external.3in b/ompi/mpi/man/man3/MPI_Pack_external.3in index 1e0771f5531..eba3f25d8bd 100644 --- a/ompi/mpi/man/man3/MPI_Pack_external.3in +++ b/ompi/mpi/man/man3/MPI_Pack_external.3in @@ -29,7 +29,7 @@ MPI_PACK_EXTERNAL(\fIDATAREP, INBUF, INCOUNT, DATATYPE, OUTBUF, OUTSIZE, POSITION, IERROR\fP) INTEGER \fIINCOUNT, DATATYPE, IERROR\fP - INTEGER (KIND=MPI_ADDRESS_KIND) \fIOUTSIZE, POSITION\fP + INTEGER(KIND=MPI_ADDRESS_KIND) \fIOUTSIZE, POSITION\fP CHARACTER*(*) \fIDATAREP\fP \fIINBUF(*), OUTBUF(*)\fP diff --git a/ompi/mpi/man/man3/MPI_Pack_external_size.3in b/ompi/mpi/man/man3/MPI_Pack_external_size.3in index 4c6f932bdd0..c6656d946d7 100644 --- a/ompi/mpi/man/man3/MPI_Pack_external_size.3in +++ b/ompi/mpi/man/man3/MPI_Pack_external_size.3in @@ -26,7 +26,7 @@ USE MPI MPI_PACK_EXTERNAL_SIZE(\fIDATAREP, INCOUNT, DATATYPE, SIZE, IERROR\fP) INTEGER \fIINCOUNT, DATATYPE, IERROR\fP - INTEGER (KIND=MPI_ADDRESS_KIND) \fISIZE\fP + INTEGER(KIND=MPI_ADDRESS_KIND) \fISIZE\fP CHARACTER*(*) \fIDATAREP\fP .fi diff --git a/ompi/mpi/man/man3/MPI_Unpack_external.3in b/ompi/mpi/man/man3/MPI_Unpack_external.3in index 378c0aac47c..5144e17cee6 100644 --- a/ompi/mpi/man/man3/MPI_Unpack_external.3in +++ b/ompi/mpi/man/man3/MPI_Unpack_external.3in @@ -29,7 +29,7 @@ MPI_UNPACK_EXTERNAL(\fIDATAREP, INBUF, INSIZE, POSITION, OUTBUF, OUTCOUNT, DATATYPE, IERROR\fP) INTEGER \fIOUTCOUNT, DATATYPE, IERROR\fP - INTEGER (KIND=MPI_ADDRESS_KIND) \fIINSIZE, POSITION\fP + INTEGER(KIND=MPI_ADDRESS_KIND) \fIINSIZE, POSITION\fP CHARACTER*(*) \fIDATAREP\fP \fIINBUF(*), OUTBUF(*)\fP diff --git a/ompi/mpi/man/man3/MPI_Win_shared_query.3in b/ompi/mpi/man/man3/MPI_Win_shared_query.3in index 237d9db1571..9b627ccbc25 100644 --- a/ompi/mpi/man/man3/MPI_Win_shared_query.3in +++ b/ompi/mpi/man/man3/MPI_Win_shared_query.3in @@ -23,7 +23,7 @@ USE MPI ! or the older form: INCLUDE 'mpif.h' MPI_WIN_SHARED_QUERY(WIN, RANK, SIZE, DISP_UNIT, BASEPTR, IERROR) INTEGER WIN, RANK, DISP_UNIT, IERROR - INTEGER (KIND=MPI_ADDRESS_KIND) SIZE, BASEPTR + INTEGER(KIND=MPI_ADDRESS_KIND) SIZE, BASEPTR .fi .SH Fortran 2008 Syntax From d3a29e364ce6831649f15c9b332e7bbeeaea545f Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 15:11:06 +0900 Subject: [PATCH 15/19] man: Change 'MPI ADDRESS KIND' to 'MPI_ADDRESS_KIND'. (underscores instead of spaces) --- ompi/mpi/man/man3/MPI_Type_get_contents.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_Type_get_contents.3in b/ompi/mpi/man/man3/MPI_Type_get_contents.3in index e32f691e749..1401980fb7f 100644 --- a/ompi/mpi/man/man3/MPI_Type_get_contents.3in +++ b/ompi/mpi/man/man3/MPI_Type_get_contents.3in @@ -96,7 +96,7 @@ The committed state of returned derived data types is undefined, that is, the da .sp Note that MPI_Type_get_contents can be invoked with a data-type argument that was constructed using MPI_Type_create_f90_real, MPI_Type_create_f90_integer, or MPI_Type_create_f90_complex (an unnamed predefined data type). In such a case, an empty \fIarray_of_datatypes\fP is returned. .sp -In the MPI-1 data-type constructor calls, the address arguments in Fortran are of type INTEGER. In the new MPI-2 calls, the address arguments are of type INTEGER(KIND=MPI ADDRESS KIND). The call MPI_Type_get_contents returns all addresses in an argument of type INTEGER(KIND=MPI ADDRESS KIND). This is true even if the old MPI-1 calls were used. Thus, the location of values returned can be thought of as being returned by the C bindings. It can also be determined by examining the new MPI-2 calls for data-type constructors for the deprecated MPI-1 calls that involve addresses. +In the MPI-1 data-type constructor calls, the address arguments in Fortran are of type INTEGER. In the new MPI-2 calls, the address arguments are of type INTEGER(KIND=MPI_ADDRESS_KIND). The call MPI_Type_get_contents returns all addresses in an argument of type INTEGER(KIND=MPI_ADDRESS_KIND). This is true even if the old MPI-1 calls were used. Thus, the location of values returned can be thought of as being returned by the C bindings. It can also be determined by examining the new MPI-2 calls for data-type constructors for the deprecated MPI-1 calls that involve addresses. .SH FORTRAN 77 NOTES .ft R From 7fef70c9b88a5f802e1b0f52b8fc33fea6ce5801 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Mon, 19 Oct 2015 15:24:12 +0900 Subject: [PATCH 16/19] man: Add `MPI_DIST_GRAPH` in `MPI_TOPO_TEST` manpage. `MPI_DIST_GRAPH` was added in MPI-2.2. --- ompi/mpi/man/man3/MPI_Topo_test.3in | 1 + 1 file changed, 1 insertion(+) diff --git a/ompi/mpi/man/man3/MPI_Topo_test.3in b/ompi/mpi/man/man3/MPI_Topo_test.3in index c3c9ab72c33..4d4b7bbcd2f 100644 --- a/ompi/mpi/man/man3/MPI_Topo_test.3in +++ b/ompi/mpi/man/man3/MPI_Topo_test.3in @@ -65,6 +65,7 @@ The output value \fItop_type\fP is one of the following: .nf MPI_GRAPH graph topology MPI_CART Cartesian topology + MPI_DIST_GRAPH distributed graph topology MPI_UNDEFINED no topology .fi From d9c93c9f5d71c7e8cecedb2bed39d8d33029baf1 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Tue, 20 Oct 2015 08:49:10 +0900 Subject: [PATCH 17/19] man: Add `const` that is removed accidentally in 2226cdb. --- ompi/mpi/man/man3/MPI_File_iwrite_shared.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in index 58e3c539424..60f5dd57273 100644 --- a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in +++ b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in @@ -15,7 +15,7 @@ .nf C Syntax #include - int MPI_File_iwrite_shared(MPI_File \fIfh\fP, void \fI*buf\fP, int \fIcount\fP, + int MPI_File_iwrite_shared(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP) .fi From 373a94a3f1806d4b1b7180cc2369471a5dc71021 Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Tue, 20 Oct 2015 14:41:22 +0900 Subject: [PATCH 18/19] man: Revert my MPI_File_iwrite_shared manpage change. This reverts commit 2226cdb3dacd21dcf7e9836db57ddb9d7936ac43 and d9c93c9f5d71c7e8cecedb2bed39d8d33029baf1. Because Gilles' b17c89c1 committed a few hours ago has the same change, my RP branch had a conflict. --- ompi/mpi/man/man3/MPI_File_iwrite_shared.3in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in index 60f5dd57273..8391a777eae 100644 --- a/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in +++ b/ompi/mpi/man/man3/MPI_File_iwrite_shared.3in @@ -15,14 +15,14 @@ .nf C Syntax #include - int MPI_File_iwrite_shared(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP, - MPI_Datatype \fIdatatype\fP, MPI_Request \fI*request\fP) + int MPI_File_(MPI_File \fIfh\fP, const void \fI*buf\fP, int \fIcount\fP, MPI_Datatype + \fIdatatype\fP, MPI_Request \fI*request\fP) .fi .SH Fortran Syntax .nf INCLUDE 'mpif.h' - MPI_FILE_IWRITE_SHARED(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP,\fI IERROR\fP) + MPI_File_(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, \fIDATATYPE\fP, \fIREQUEST\fP,\fI IERROR\fP) BUF(*) INTEGER FH, COUNT, DATATYPE, REQUEST, IERROR From 7ab464fbb46f9a9cd6f70502d7c0da6a8046914c Mon Sep 17 00:00:00 2001 From: KAWASHIMA Takahiro Date: Tue, 20 Oct 2015 15:32:45 +0900 Subject: [PATCH 19/19] Revert "man: Remove unnecessary spaces in front of parameters." This reverts commit 3253a30ab2922fa9eef2e0b5e7843a949a650853. Because Gilles' b17c89c1 committed a few hours ago has the same change, my RP branch had a conflict. --- ompi/mpi/man/man3/MPI_File_read.3in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ompi/mpi/man/man3/MPI_File_read.3in b/ompi/mpi/man/man3/MPI_File_read.3in index b46c7fee781..7c19ee60350 100644 --- a/ompi/mpi/man/man3/MPI_File_read.3in +++ b/ompi/mpi/man/man3/MPI_File_read.3in @@ -19,7 +19,7 @@ C Syntax .SH Fortran Syntax .nf INCLUDE 'mpif.h' - MPI_FILE_READ(\fIFH\fP, \fIBUF\fP, \fICOUNT\fP, + MPI_FILE_READ(\fI FH\fP, \fI BUF\fP, \fICOUNT\fP, \fI DATATYPE\fP, \fISTATUS\fP, \fIIERROR\fP) BUF(*) INTEGER FH, COUNT, DATATYPE, STATUS(MPI_STATUS_SIZE),