Skip to content

Commit

Permalink
Merge pull request #4766 from minsii/pr/inline-cleanup
Browse files Browse the repository at this point in the history 
Cleanup static inline for fast paths and add configure option to control force inline

Approved-by: Hui Zhou <hzhou321@anl.gov>
  • Loading branch information
@minsii
minsii committed Sep 3, 2020
2 parents e02345d + cff2651 commit 404cd8a
Show file tree
Hide file tree
Showing 38 changed files with 1,368 additions and 1,137 deletions.
5 changes: 4 additions & 1 deletion configure.ac
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,7 +415,8 @@ AC_ARG_ENABLE(fast,
MPICHLIB_FFLAGS, and MPICHLIB_FCFLAGS. (default is -O2)
ndebug - Appends -DNDEBUG to MPICHLIB_CFLAGS.
no-strict-alignment - relax strict alignment requirement
all|yes - "O2" and "ndebug" are enabled
alwaysinline - Force compiler to always inline performance critical routines
all|yes - "O2", "ndebug", and "alwaysinline" are enabled
none - None of above options, i.e. --disable-fast
],,enable_fast=O2)

Expand Down Expand Up @@ -917,6 +918,8 @@ for option in $enable_fast ; do
no-strict-alignment)
enable_fast_no_strict_alignment=yes
;;
alwaysinline) # No op in MPICH. See mpl/configure.ac
;;
all|yes)
enable_fast_ndebug=yes
enable_fast_opts=O2
Expand Down
20 changes: 11 additions & 9 deletions src/mpid/ch4/generic/am/mpidig_am_recv.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,10 @@
#include "ch4r_proc.h"
#include "ch4r_recv.h"

static inline void MPIDIG_prepare_recv_req(int rank, int tag, MPIR_Context_id_t context_id,
void *buf, MPI_Aint count, MPI_Datatype datatype,
MPIR_Request * rreq)
MPL_STATIC_INLINE_PREFIX void MPIDIG_prepare_recv_req(int rank, int tag,
MPIR_Context_id_t context_id, void *buf,
MPI_Aint count, MPI_Datatype datatype,
MPIR_Request * rreq)
{
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDIG_PREPARE_RECV_REQ);
MPIR_FUNC_VERBOSE_ENTER(MPID_STATE_MPIDIG_PREPARE_RECV_REQ);
Expand All @@ -27,9 +28,9 @@ static inline void MPIDIG_prepare_recv_req(int rank, int tag, MPIR_Context_id_t
MPIR_FUNC_VERBOSE_EXIT(MPID_STATE_MPIDIG_PREPARE_RECV_REQ);
}

static inline int MPIDIG_handle_unexpected(void *buf, MPI_Aint count, MPI_Datatype datatype,
MPIR_Comm * comm, int context_offset,
MPIR_Request * rreq)
MPL_STATIC_INLINE_PREFIX int MPIDIG_handle_unexpected(void *buf, MPI_Aint count,
MPI_Datatype datatype, MPIR_Comm * comm,
int context_offset, MPIR_Request * rreq)
{
int mpi_errno = MPI_SUCCESS;
int dt_contig;
Expand Down Expand Up @@ -115,9 +116,10 @@ static inline int MPIDIG_handle_unexpected(void *buf, MPI_Aint count, MPI_Dataty
goto fn_exit;
}

static inline int MPIDIG_do_irecv(void *buf, MPI_Aint count, MPI_Datatype datatype, int rank,
int tag, MPIR_Comm * comm, int context_offset,
MPIR_Request ** request, int alloc_req, uint64_t flags)
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_irecv(void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm,
int context_offset, MPIR_Request ** request,
int alloc_req, uint64_t flags)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *rreq = NULL, *unexp_req = NULL;
Expand Down
67 changes: 35 additions & 32 deletions src/mpid/ch4/generic/am/mpidig_am_send.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

#define MPIDIG_AM_SEND_HDR_SIZE sizeof(MPIDIG_hdr_t)

static inline int mpidig_eager_limit(int is_local)
MPL_STATIC_INLINE_PREFIX int mpidig_eager_limit(int is_local)
{
if (MPIR_CVAR_CH4_EAGER_MAX_MSG_SIZE > 0) {
return MPIR_CVAR_CH4_EAGER_MAX_MSG_SIZE;
Expand All @@ -51,24 +51,26 @@ static inline int mpidig_eager_limit(int is_local)
return thresh;
}

static inline int MPIDIG_do_eager_send(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag);
static inline int MPIDIG_do_rndv_send(const void *buf, MPI_Aint count, MPI_Datatype datatype,
MPI_Aint data_sz,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag);
static inline int MPIDIG_do_ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag);

static inline int MPIDIG_isend_impl(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
int type, MPIR_Errflag_t errflag)
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_eager_send(const void *buf, MPI_Aint count,
MPI_Datatype datatype, int rank, int tag,
MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag);
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_rndv_send(const void *buf, MPI_Aint count,
MPI_Datatype datatype, MPI_Aint data_sz, int rank,
int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag);
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm,
int context_offset, MPIDI_av_entry_t * addr,
MPIR_Request ** request, MPIR_Errflag_t errflag);

MPL_STATIC_INLINE_PREFIX int MPIDIG_isend_impl(const void *buf, MPI_Aint count,
MPI_Datatype datatype, int rank, int tag,
MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
int type, MPIR_Errflag_t errflag)
{
MPI_Aint data_sz;
MPIR_Datatype *dt_ptr;
Expand All @@ -87,10 +89,11 @@ static inline int MPIDIG_isend_impl(const void *buf, MPI_Aint count, MPI_Datatyp
}
}

static inline int MPIDIG_do_eager_send(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag)
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_eager_send(const void *buf, MPI_Aint count,
MPI_Datatype datatype, int rank, int tag,
MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *sreq = *request;
Expand Down Expand Up @@ -135,10 +138,10 @@ static inline int MPIDIG_do_eager_send(const void *buf, MPI_Aint count, MPI_Data
goto fn_exit;
}

static inline int MPIDIG_do_ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag)
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_ssend(const void *buf, MPI_Aint count, MPI_Datatype datatype,
int rank, int tag, MPIR_Comm * comm,
int context_offset, MPIDI_av_entry_t * addr,
MPIR_Request ** request, MPIR_Errflag_t errflag)
{
int mpi_errno = MPI_SUCCESS, c;
MPIR_Request *sreq = *request;
Expand Down Expand Up @@ -188,11 +191,11 @@ static inline int MPIDIG_do_ssend(const void *buf, MPI_Aint count, MPI_Datatype
goto fn_exit;
}

static inline int MPIDIG_do_rndv_send(const void *buf, MPI_Aint count, MPI_Datatype datatype,
MPI_Aint data_sz,
int rank, int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag)
MPL_STATIC_INLINE_PREFIX int MPIDIG_do_rndv_send(const void *buf, MPI_Aint count,
MPI_Datatype datatype, MPI_Aint data_sz, int rank,
int tag, MPIR_Comm * comm, int context_offset,
MPIDI_av_entry_t * addr, MPIR_Request ** request,
MPIR_Errflag_t errflag)
{
int mpi_errno = MPI_SUCCESS;
MPIR_Request *sreq = *request;
Expand Down
2 changes: 1 addition & 1 deletion src/mpid/ch4/netmod/ofi/catalog.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
* See COPYRIGHT in top-level directory
*/

static inline void MPIDI_OFI_unused_gen_catalog()
static void MPIDI_OFI_unused_gen_catalog()
{
#if 0
char *a;
Expand Down
72 changes: 38 additions & 34 deletions src/mpid/ch4/netmod/ofi/ofi_am.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,25 +10,26 @@
#include "ofi_am_events.h"
#include "mpidu_genq.h"

static inline int MPIDI_OFI_progress_do_queue(int vni_idx);
MPL_STATIC_INLINE_PREFIX int MPIDI_OFI_progress_do_queue(int vni_idx);

static inline void MPIDI_NM_am_request_init(MPIR_Request * req)
MPL_STATIC_INLINE_PREFIX void MPIDI_NM_am_request_init(MPIR_Request * req)
{
MPIDI_OFI_AMREQUEST(req, req_hdr) = NULL;
}

static inline void MPIDI_NM_am_request_finalize(MPIR_Request * req)
MPL_STATIC_INLINE_PREFIX void MPIDI_NM_am_request_finalize(MPIR_Request * req)
{
MPIDI_OFI_am_clear_request(req);
}

static inline int MPIDI_NM_am_isend(int rank,
MPIR_Comm * comm,
int handler_id,
const void *am_hdr,
size_t am_hdr_sz,
const void *data,
MPI_Count count, MPI_Datatype datatype, MPIR_Request * sreq)
MPL_STATIC_INLINE_PREFIX int MPIDI_NM_am_isend(int rank,
MPIR_Comm * comm,
int handler_id,
const void *am_hdr,
size_t am_hdr_sz,
const void *data,
MPI_Count count, MPI_Datatype datatype,
MPIR_Request * sreq)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_NM_AM_ISEND);
Expand All @@ -41,13 +42,14 @@ static inline int MPIDI_NM_am_isend(int rank,
return mpi_errno;
}

static inline int MPIDI_NM_am_isendv(int rank,
MPIR_Comm * comm,
int handler_id,
struct iovec *am_hdr,
size_t iov_len,
const void *data,
MPI_Count count, MPI_Datatype datatype, MPIR_Request * sreq)
MPL_STATIC_INLINE_PREFIX int MPIDI_NM_am_isendv(int rank,
MPIR_Comm * comm,
int handler_id,
struct iovec *am_hdr,
size_t iov_len,
const void *data,
MPI_Count count, MPI_Datatype datatype,
MPIR_Request * sreq)
{
int mpi_errno = MPI_SUCCESS, is_allocated;
size_t am_hdr_sz = 0, i;
Expand Down Expand Up @@ -89,14 +91,14 @@ static inline int MPIDI_NM_am_isendv(int rank,
return mpi_errno;
}

static inline int MPIDI_NM_am_isend_reply(MPIR_Context_id_t context_id,
int src_rank,
int handler_id,
const void *am_hdr,
size_t am_hdr_sz,
const void *data,
MPI_Count count,
MPI_Datatype datatype, MPIR_Request * sreq)
MPL_STATIC_INLINE_PREFIX int MPIDI_NM_am_isend_reply(MPIR_Context_id_t context_id,
int src_rank,
int handler_id,
const void *am_hdr,
size_t am_hdr_sz,
const void *data,
MPI_Count count,
MPI_Datatype datatype, MPIR_Request * sreq)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_NM_AM_ISEND_REPLY);
Expand All @@ -109,7 +111,7 @@ static inline int MPIDI_NM_am_isend_reply(MPIR_Context_id_t context_id,
return mpi_errno;
}

static inline size_t MPIDI_NM_am_hdr_max_sz(void)
MPL_STATIC_INLINE_PREFIX size_t MPIDI_NM_am_hdr_max_sz(void)
{
/* Maximum size that fits in short send */
size_t max_shortsend = MPIDI_OFI_DEFAULT_SHORT_SEND_SIZE -
Expand All @@ -120,19 +122,20 @@ static inline size_t MPIDI_NM_am_hdr_max_sz(void)
return MPL_MIN(max_shortsend, max_representable);
}

static inline size_t MPIDI_NM_am_eager_limit(void)
MPL_STATIC_INLINE_PREFIX size_t MPIDI_NM_am_eager_limit(void)
{
return MPIDI_OFI_DEFAULT_SHORT_SEND_SIZE - sizeof(MPIDI_OFI_am_header_t);
}

static inline size_t MPIDI_NM_am_eager_buf_limit(void)
MPL_STATIC_INLINE_PREFIX size_t MPIDI_NM_am_eager_buf_limit(void)
{
return MPIDI_OFI_DEFAULT_SHORT_SEND_SIZE;
}

static inline int MPIDI_NM_am_send_hdr(int rank,
MPIR_Comm * comm,
int handler_id, const void *am_hdr, size_t am_hdr_sz)
MPL_STATIC_INLINE_PREFIX int MPIDI_NM_am_send_hdr(int rank,
MPIR_Comm * comm,
int handler_id, const void *am_hdr,
size_t am_hdr_sz)
{
int mpi_errno = MPI_SUCCESS;
MPIR_FUNC_VERBOSE_STATE_DECL(MPID_STATE_MPIDI_NM_AM_SEND_HDR);
Expand All @@ -148,9 +151,10 @@ static inline int MPIDI_NM_am_send_hdr(int rank,
goto fn_exit;
}

static inline int MPIDI_NM_am_send_hdr_reply(MPIR_Context_id_t context_id,
int src_rank,
int handler_id, const void *am_hdr, size_t am_hdr_sz)
MPL_STATIC_INLINE_PREFIX int MPIDI_NM_am_send_hdr_reply(MPIR_Context_id_t context_id,
int src_rank,
int handler_id, const void *am_hdr,
size_t am_hdr_sz)
{
int mpi_errno = MPI_SUCCESS;

Expand Down

0 comments on commit 404cd8a

Please sign in to comment.