From 06ef3446309d31418bcc163e4077ff0c5663cdb8 Mon Sep 17 00:00:00 2001 From: Valentin Petrov Date: Fri, 6 Oct 2017 10:24:41 +0300 Subject: [PATCH 1/2] coll/hcoll: extends dtypes support Adds support for legacy MPI_UB/LB types (old apps may use it) as well as for BOOL/WCHAR. Signed-off-by: Valentin Petrov --- ompi/mca/coll/hcoll/coll_hcoll_dtypes.h | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h index f0efb41c4fd..d834308fe56 100644 --- a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h +++ b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h @@ -36,8 +36,16 @@ total 15 types static dte_data_representation_t* ompi_datatype_2_dte_data_rep[OMPI_DATATYPE_MAX_PREDEFINED] = { &DTE_ZERO, /*OPAL_DATATYPE_LOOP 0 */ &DTE_ZERO, /*OPAL_DATATYPE_END_LOOP 1 */ - &DTE_ZERO, /*OPAL_DATATYPE_LB 2 */ - &DTE_ZERO, /*OPAL_DATATYPE_UB 3 */ +#if defined(DTE_LB) + &DTE_LB, /*OPAL_DATATYPE_LB 2 */ +#else + &DTE_ZERO, +#endif +#if defined(DTE_UB) + &DTE_UB, /*OPAL_DATATYPE_UB 3 */ +#else + &DTE_ZERO, +#endif &DTE_BYTE, /*OPAL_DATATYPE_INT1 4 */ &DTE_INT16, /*OPAL_DATATYPE_INT2 5 */ &DTE_INT32, /*OPAL_DATATYPE_INT4 6 */ @@ -68,8 +76,16 @@ static dte_data_representation_t* ompi_datatype_2_dte_data_rep[OMPI_DATATYPE_MAX #else &DTE_ZERO, #endif - &DTE_ZERO, /*OPAL_DATATYPE_BOOL 22 */ - &DTE_ZERO, /*OPAL_DATATYPE_WCHAR 23 */ +#if defined(DTE_BOOL) + &DTE_BOOL, /*OPAL_DATATYPE_BOOL 22 */ +#else + &DTE_ZERO, +#endif +#if defined(DTE_WCHAR) + &DTE_WCHAR, /*OPAL_DATATYPE_WCHAR 23 */ +#else + &DTE_ZERO, +#endif &DTE_ZERO /*OPAL_DATATYPE_UNAVAILABLE 24 */ }; From 1e311b2619f2bd6c177230a830ade6be89aa4d65 Mon Sep 17 00:00:00 2001 From: Valentin Petrov Date: Fri, 6 Oct 2017 10:25:47 +0300 Subject: [PATCH 2/2] coll/hcoll: dtype fallback optimization If hcoll fails to create mpi derived type let's set zero_dte on this dtype. This will save cycles on subsequent collective calls with the same derived type since we will not try to create hcoll type again. Signed-off-by: Valentin Petrov --- ompi/mca/coll/hcoll/coll_hcoll.h | 1 + ompi/mca/coll/hcoll/coll_hcoll_dtypes.h | 17 +++++++++++++---- ompi/mca/coll/hcoll/coll_hcoll_module.c | 3 ++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ompi/mca/coll/hcoll/coll_hcoll.h b/ompi/mca/coll/hcoll/coll_hcoll.h index 6e8382d49f5..1ad34be11c6 100644 --- a/ompi/mca/coll/hcoll/coll_hcoll.h +++ b/ompi/mca/coll/hcoll/coll_hcoll.h @@ -56,6 +56,7 @@ typedef struct { } mca_coll_hcoll_dtype_t; OBJ_CLASS_DECLARATION(mca_coll_hcoll_dtype_t); +extern mca_coll_hcoll_dtype_t zero_dte_mapping; struct mca_coll_hcoll_component_t { /** Base coll component */ mca_coll_base_component_2_0_0_t super; diff --git a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h index d834308fe56..a818e6675ba 100644 --- a/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h +++ b/ompi/mca/coll/hcoll/coll_hcoll_dtypes.h @@ -10,7 +10,7 @@ #include "ompi/mca/op/op.h" #include "hcoll/api/hcoll_dte.h" extern int hcoll_type_attr_keyval; - +extern mca_coll_hcoll_dtype_t zero_dte_mapping; /*to keep this at hand: Ids of the basic opal_datatypes: #define OPAL_DATATYPE_INT1 4 #define OPAL_DATATYPE_INT2 5 @@ -97,15 +97,21 @@ enum { #if HCOLL_API >= HCOLL_VERSION(3,6) static inline -int hcoll_map_derived_type(ompi_datatype_t *dtype, dte_data_representation_t *new_dte) +void hcoll_map_derived_type(ompi_datatype_t *dtype, dte_data_representation_t *new_dte) { int rc; if (NULL == dtype->args) { /* predefined type, shouldn't call this */ - return OMPI_SUCCESS; + return; } rc = hcoll_create_mpi_type((void*)dtype, new_dte); - return rc == HCOLL_SUCCESS ? OMPI_SUCCESS : OMPI_ERROR; + if (rc != HCOLL_SUCCESS) { + /* If hcoll fails to create mpi derived type let's set zero_dte on this dtype. + This will save cycles on subsequent collective calls with the same derived + type since we will not try to create hcoll type again. */ + ompi_attr_set_c(TYPE_ATTR, (void*)dtype, &(dtype->d_keyhash), + hcoll_type_attr_keyval, &zero_dte_mapping, false); + } } static dte_data_representation_t find_derived_mapping(ompi_datatype_t *dtype){ @@ -238,6 +244,9 @@ static int hcoll_type_attr_del_fn(MPI_Datatype type, int keyval, void *attr_val, (mca_coll_hcoll_dtype_t*) attr_val; assert(dtype); + if (&zero_dte_mapping == dtype) { + return OMPI_SUCCESS; + } if (HCOLL_SUCCESS != (ret = hcoll_dt_destroy(dtype->type))) { HCOL_ERROR("failed to delete type attr: hcoll_dte_destroy returned %d",ret); return OMPI_ERROR; diff --git a/ompi/mca/coll/hcoll/coll_hcoll_module.c b/ompi/mca/coll/hcoll/coll_hcoll_module.c index dfc8f676727..1cd36fd89b5 100644 --- a/ompi/mca/coll/hcoll/coll_hcoll_module.c +++ b/ompi/mca/coll/hcoll/coll_hcoll_module.c @@ -17,7 +17,7 @@ int hcoll_comm_attr_keyval; int hcoll_type_attr_keyval; - +mca_coll_hcoll_dtype_t zero_dte_mapping; /* * Initial query function that is invoked during MPI_INIT, allowing * this module to indicate what level of thread support it provides. @@ -333,6 +333,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority) } if (mca_coll_hcoll_component.derived_types_support_enabled) { + zero_dte_mapping.type = DTE_ZERO; copy_fn.attr_datatype_copy_fn = (MPI_Type_internal_copy_attr_function *) MPI_TYPE_NULL_COPY_FN; del_fn.attr_datatype_delete_fn = hcoll_type_attr_del_fn; err = ompi_attr_create_keyval(TYPE_ATTR, copy_fn, del_fn, &hcoll_type_attr_keyval, NULL ,0, NULL);