From f13f29117d13a8d580e0904fadd8a79d172eef5e Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Tue, 28 May 2024 17:48:31 +0300 Subject: [PATCH 1/4] UCX: Propagate MPI serialized for all worker creations Move MPI to UCX thread mode function to common source. Also use serialized mode for all oshmem initializations. Signed-off-by: Thomas Vegas (cherry picked from commit 44cc663dffd42402ec694b277eaebb6a1c18ad9e) --- ompi/mca/pml/ucx/pml_ucx.c | 3 ++- opal/mca/common/ucx/common_ucx.c | 20 ++++++++++++++++++++ opal/mca/common/ucx/common_ucx.h | 1 + oshmem/mca/spml/ucx/spml_ucx.c | 5 ++++- 4 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ompi/mca/pml/ucx/pml_ucx.c b/ompi/mca/pml/ucx/pml_ucx.c index 79e2d149abb..a0b3d2bf827 100644 --- a/ompi/mca/pml/ucx/pml_ucx.c +++ b/ompi/mca/pml/ucx/pml_ucx.c @@ -293,7 +293,8 @@ int mca_pml_ucx_init(int enable_mpi_threads) if (enable_mpi_threads) { params.thread_mode = UCS_THREAD_MODE_MULTI; } else { - params.thread_mode = UCS_THREAD_MODE_SINGLE; + params.thread_mode = + opal_common_ucx_thread_mode(ompi_mpi_thread_provided); } #if HAVE_DECL_UCP_WORKER_FLAG_IGNORE_REQUEST_LEAK diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c index 1cd250e8944..23d24f64930 100644 --- a/opal/mca/common/ucx/common_ucx.c +++ b/opal/mca/common/ucx/common_ucx.c @@ -25,7 +25,10 @@ #include "opal/util/argv.h" #include "opal/util/printf.h" +#include "mpi.h" + #include + #include #include @@ -49,6 +52,23 @@ static void opal_common_ucx_mem_release_cb(void *buf, size_t length, ucm_vm_munmap(buf, length); } +ucs_thread_mode_t opal_common_ucx_thread_mode(int ompi_mode) +{ + switch (ompi_mode) { + case MPI_THREAD_MULTIPLE: + return UCS_THREAD_MODE_MULTI; + case MPI_THREAD_SERIALIZED: + return UCS_THREAD_MODE_SERIALIZED; + case MPI_THREAD_FUNNELED: + case MPI_THREAD_SINGLE: + return UCS_THREAD_MODE_SINGLE; + default: + MCA_COMMON_UCX_WARN("Unknown MPI thread mode %d, using multithread", + ompi_mode); + return UCS_THREAD_MODE_MULTI; + } +} + OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *component) { char *default_tls = "rc_verbs,ud_verbs,rc_mlx5,dc_mlx5,ud_mlx5,cuda_ipc,rocm_ipc"; diff --git a/opal/mca/common/ucx/common_ucx.h b/opal/mca/common/ucx/common_ucx.h index 4b78bc66587..afd322b9add 100644 --- a/opal/mca/common/ucx/common_ucx.h +++ b/opal/mca/common/ucx/common_ucx.h @@ -124,6 +124,7 @@ OPAL_DECLSPEC int opal_common_ucx_del_procs(opal_common_ucx_del_proc_t *procs, s OPAL_DECLSPEC int opal_common_ucx_del_procs_nofence(opal_common_ucx_del_proc_t *procs, size_t count, size_t my_rank, size_t max_disconnect, ucp_worker_h worker); OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *component); +OPAL_DECLSPEC ucs_thread_mode_t opal_common_ucx_thread_mode(int ompi_mode); static inline ucs_status_t opal_common_ucx_request_status(ucs_status_ptr_t request) diff --git a/oshmem/mca/spml/ucx/spml_ucx.c b/oshmem/mca/spml/ucx/spml_ucx.c index 21a1d817bc1..33bbda01f27 100644 --- a/oshmem/mca/spml/ucx/spml_ucx.c +++ b/oshmem/mca/spml/ucx/spml_ucx.c @@ -1016,8 +1016,11 @@ static int mca_spml_ucx_ctx_create_common(long options, mca_spml_ucx_ctx_t **ucx ucx_ctx->strong_sync = mca_spml_ucx_ctx_default.strong_sync; params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; - if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || options & SHMEM_CTX_PRIVATE || options & SHMEM_CTX_SERIALIZED) { + if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || + oshmem_mpi_thread_provided == SHMEM_THREAD_FUNNELED || options & SHMEM_CTX_PRIVATE) { params.thread_mode = UCS_THREAD_MODE_SINGLE; + } else if (oshmem_mpi_thread_provided == SHMEM_THREAD_SERIALIZED || options & SHMEM_CTX_SERIALIZED) { + params.thread_mode = UCS_THREAD_MODE_SERIALIZED; } else { params.thread_mode = UCS_THREAD_MODE_MULTI; } From 6753f8debb9f6083632b4b556b4fdc3402e37e20 Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Tue, 11 Jun 2024 09:46:26 +0300 Subject: [PATCH 2/4] UCX: Propagate MPI serialized for all worker creations Move MPI to UCX thread mode function to common source. Also use serialized mode for all oshmem initializations. Signed-off-by: Thomas Vegas (cherry picked from commit 44cc663dffd42402ec694b277eaebb6a1c18ad9e) --- ompi/mca/osc/ucx/osc_ucx_component.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ompi/mca/osc/ucx/osc_ucx_component.c b/ompi/mca/osc/ucx/osc_ucx_component.c index a6e4fe661e1..f4513b6bf5f 100644 --- a/ompi/mca/osc/ucx/osc_ucx_component.c +++ b/ompi/mca/osc/ucx/osc_ucx_component.c @@ -415,8 +415,12 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in assert(mca_osc_ucx_component.ucp_worker == NULL); memset(&worker_params, 0, sizeof(worker_params)); worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; - worker_params.thread_mode = (mca_osc_ucx_component.enable_mpi_threads == true) - ? UCS_THREAD_MODE_MULTI : UCS_THREAD_MODE_SINGLE; + if (mca_osc_ucx_component.enable_mpi_threads == true) { + worker_params.thread_mode = UCS_THREAD_MODE_MULTI; + } else { + worker_params.thread_mode = + opal_common_ucx_thread_mode(ompi_mpi_thread_provided); + } status = ucp_worker_create(mca_osc_ucx_component.ucp_context, &worker_params, &(mca_osc_ucx_component.ucp_worker)); if (UCS_OK != status) { From 972ded5fa89f06199660701676171e0f9bf0002a Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Tue, 11 Jun 2024 17:51:22 +0300 Subject: [PATCH 3/4] UCX: Propagate MPI serialized for all worker creations Move MPI to UCX thread mode function to common source. Also use serialized mode for all oshmem initializations. Signed-off-by: Thomas Vegas (cherry picked from commit 44cc663dffd42402ec694b277eaebb6a1c18ad9e) --- ompi/mca/pml/ucx/pml_ucx.c | 1 - oshmem/mca/spml/ucx/spml_ucx_component.c | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ompi/mca/pml/ucx/pml_ucx.c b/ompi/mca/pml/ucx/pml_ucx.c index a0b3d2bf827..5eb621990c0 100644 --- a/ompi/mca/pml/ucx/pml_ucx.c +++ b/ompi/mca/pml/ucx/pml_ucx.c @@ -288,7 +288,6 @@ int mca_pml_ucx_init(int enable_mpi_threads) PML_UCX_VERBOSE(1, "mca_pml_ucx_init"); - /* TODO check MPI thread mode */ params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; if (enable_mpi_threads) { params.thread_mode = UCS_THREAD_MODE_MULTI; diff --git a/oshmem/mca/spml/ucx/spml_ucx_component.c b/oshmem/mca/spml/ucx/spml_ucx_component.c index 89c04e2f9eb..41f6581dc94 100644 --- a/oshmem/mca/spml/ucx/spml_ucx_component.c +++ b/oshmem/mca/spml/ucx/spml_ucx_component.c @@ -322,6 +322,8 @@ static int spml_ucx_init(void) wkr_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; if (oshmem_mpi_thread_requested == SHMEM_THREAD_MULTIPLE) { wkr_params.thread_mode = UCS_THREAD_MODE_MULTI; + } else if (oshmem_mpi_thread_requested == SHMEM_THREAD_SERIALIZED) { + wkr_params.thread_mode = UCS_THREAD_MODE_SERIALIZED; } else { wkr_params.thread_mode = UCS_THREAD_MODE_SINGLE; } From 8cd79c4b2248f8764f8b5dbacb896545e3a60a08 Mon Sep 17 00:00:00 2001 From: Thomas Vegas Date: Wed, 12 Jun 2024 10:09:58 +0300 Subject: [PATCH 4/4] UCX: Propagate MPI serialized for all worker creations Move MPI to UCX thread mode function to common source. Also use serialized mode for all oshmem initializations. Signed-off-by: Thomas Vegas (cherry picked from commit 44cc663dffd42402ec694b277eaebb6a1c18ad9e) --- ompi/mca/osc/ucx/osc_ucx_component.c | 2 +- opal/mca/common/ucx/common_ucx.c | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ompi/mca/osc/ucx/osc_ucx_component.c b/ompi/mca/osc/ucx/osc_ucx_component.c index f4513b6bf5f..5fc88a9e102 100644 --- a/ompi/mca/osc/ucx/osc_ucx_component.c +++ b/ompi/mca/osc/ucx/osc_ucx_component.c @@ -415,7 +415,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, in assert(mca_osc_ucx_component.ucp_worker == NULL); memset(&worker_params, 0, sizeof(worker_params)); worker_params.field_mask = UCP_WORKER_PARAM_FIELD_THREAD_MODE; - if (mca_osc_ucx_component.enable_mpi_threads == true) { + if (mca_osc_ucx_component.enable_mpi_threads) { worker_params.thread_mode = UCS_THREAD_MODE_MULTI; } else { worker_params.thread_mode = diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c index 23d24f64930..b953fcdcf27 100644 --- a/opal/mca/common/ucx/common_ucx.c +++ b/opal/mca/common/ucx/common_ucx.c @@ -28,7 +28,6 @@ #include "mpi.h" #include - #include #include