diff --git a/ompi/dpm/dpm.c b/ompi/dpm/dpm.c index 647ec4e15bf..ad7f42a0770 100644 --- a/ompi/dpm/dpm.c +++ b/ompi/dpm/dpm.c @@ -595,6 +595,11 @@ int ompi_dpm_disconnect(ompi_communicator_t *comm) } else { ret = opal_pmix.disconnect(&coll); } + if (OMPI_SUCCESS != ret) { + OMPI_ERROR_LOG(ret); + OPAL_LIST_DESTRUCT(&coll); + return ret; + } OPAL_LIST_DESTRUCT(&coll); return ret; diff --git a/ompi/mca/bml/r2/bml_r2_ft.c b/ompi/mca/bml/r2/bml_r2_ft.c index 95fc6ade66b..8dc45d4f1e3 100644 --- a/ompi/mca/bml/r2/bml_r2_ft.c +++ b/ompi/mca/bml/r2/bml_r2_ft.c @@ -155,7 +155,10 @@ int mca_bml_r2_ft_event(int state) * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ - opal_pmix.fence(NULL, 0); + if( OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "bml:r2: ft_event(Restart): Failed to fence complete\n"); + return ret; + } /* * Re-open the BTL framework to get the full list of components. @@ -224,7 +227,10 @@ int mca_bml_r2_ft_event(int state) * Barrier to make all processes have been successfully restarted before * we try to remove some restart only files. */ - opal_pmix.fence(NULL, 0); + if( OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "bml:r2: ft_event(Restart): Failed to fence complete\n"); + return ret; + } /* * Re-open the BTL framework to get the full list of components. diff --git a/ompi/mca/crcp/bkmrk/crcp_bkmrk_pml.c b/ompi/mca/crcp/bkmrk/crcp_bkmrk_pml.c index 7d5e480095a..3276df26d0a 100644 --- a/ompi/mca/crcp/bkmrk/crcp_bkmrk_pml.c +++ b/ompi/mca/crcp/bkmrk/crcp_bkmrk_pml.c @@ -3028,7 +3028,10 @@ ompi_crcp_base_pml_state_t* ompi_crcp_bkmrk_pml_ft_event( if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCPBR0); - opal_pmix.fence(NULL, 0); + if( OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + exit_status = ret; + goto DONE; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP0); @@ -3096,7 +3099,10 @@ ompi_crcp_base_pml_state_t* ompi_crcp_bkmrk_pml_ft_event( if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_COREBR1); - opal_pmix.fence(NULL, 0); + if( OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + exit_status = ret; + goto DONE; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CORE2); } @@ -6207,14 +6213,16 @@ static void clear_timers(void) { static void display_all_timers(int state) { bool report_ready = false; double barrier_start, barrier_stop; - int i; + int i, ret; if( 0 != OMPI_PROC_MY_NAME->vpid ) { if( 2 > timing_enabled ) { return; } else if( 2 == timing_enabled ) { - opal_pmix.fence(NULL, 0); + if( OPAL_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + OPAL_ERROR_LOG(ret); + } return; } } @@ -6235,7 +6243,9 @@ static void display_all_timers(int state) { if( timing_enabled >= 2) { barrier_start = get_time(); - opal_pmix.fence(NULL, 0); + if( OPAL_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + OPAL_ERROR_LOG(ret); + } barrier_stop = get_time(); opal_output(0, "crcp:bkmrk: timing(%20s): %20s = %10.2f s\n", diff --git a/ompi/mca/pml/bfo/pml_bfo.c b/ompi/mca/pml/bfo/pml_bfo.c index e3a1beb447a..ce33b0d57be 100644 --- a/ompi/mca/pml/bfo/pml_bfo.c +++ b/ompi/mca/pml/bfo/pml_bfo.c @@ -666,7 +666,10 @@ int mca_pml_bfo_ft_event( int state ) if(OPAL_CRS_CHECKPOINT == state) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCPBR1); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P0); @@ -677,7 +680,10 @@ int mca_pml_bfo_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_COREBR0); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P2); } @@ -777,7 +783,10 @@ int mca_pml_bfo_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR1); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P3); } @@ -787,7 +796,10 @@ int mca_pml_bfo_ft_event( int state ) * Exchange the modex information once again. * BTLs will have republished their modex information. */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } /* * Startup the PML stack now that the modex is running again @@ -799,7 +811,10 @@ int mca_pml_bfo_ft_event( int state ) } /* Is this barrier necessary ? JJH */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } if( NULL != procs ) { for(p = 0; p < (int)num_procs; ++p) { @@ -812,7 +827,10 @@ int mca_pml_bfo_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR2); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP1); } @@ -825,7 +843,10 @@ int mca_pml_bfo_ft_event( int state ) * Exchange the modex information once again. * BTLs will have republished their modex information. */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } /* * Startup the PML stack now that the modex is running again @@ -837,7 +858,10 @@ int mca_pml_bfo_ft_event( int state ) } /* Is this barrier necessary ? JJH */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:bfo: ft_event(Restart): Failed to fence complete"); + return ret; + } if( NULL != procs ) { for(p = 0; p < (int)num_procs; ++p) { diff --git a/ompi/mca/pml/ob1/pml_ob1.c b/ompi/mca/pml/ob1/pml_ob1.c index 1f35a2d5663..5a14b52e47e 100644 --- a/ompi/mca/pml/ob1/pml_ob1.c +++ b/ompi/mca/pml/ob1/pml_ob1.c @@ -807,7 +807,10 @@ int mca_pml_ob1_ft_event( int state ) if(OPAL_CRS_CHECKPOINT == state) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCPBR1); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P0); @@ -818,7 +821,10 @@ int mca_pml_ob1_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_COREBR0); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P2); } @@ -918,13 +924,19 @@ int mca_pml_ob1_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR1); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2P3); } if (opal_cr_continue_like_restart && !first_continue_pass) { - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } /* * Startup the PML stack now that the modex is running again @@ -936,7 +948,10 @@ int mca_pml_ob1_ft_event( int state ) } /* Is this barrier necessary ? JJH */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } if( NULL != procs ) { for(p = 0; p < (int)num_procs; ++p) { @@ -949,7 +964,10 @@ int mca_pml_ob1_ft_event( int state ) if( !first_continue_pass ) { if( opal_cr_timing_barrier_enabled ) { OPAL_CR_SET_TIMER(OPAL_CR_TIMER_P2PBR2); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } } OPAL_CR_SET_TIMER(OPAL_CR_TIMER_CRCP1); } @@ -962,7 +980,10 @@ int mca_pml_ob1_ft_event( int state ) * Exchange the modex information once again. * BTLs will have republished their modex information. */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } /* * Startup the PML stack now that the modex is running again @@ -974,7 +995,10 @@ int mca_pml_ob1_ft_event( int state ) } /* Is this barrier necessary ? JJH */ - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + opal_output(0, "pml:ob1: ft_event(Restart): Failed to fence complete"); + return ret; + } if( NULL != procs ) { for(p = 0; p < (int)num_procs; ++p) { diff --git a/ompi/mca/pml/yalla/pml_yalla.c b/ompi/mca/pml/yalla/pml_yalla.c index 7890293c330..03bb65d420d 100644 --- a/ompi/mca/pml/yalla/pml_yalla.c +++ b/ompi/mca/pml/yalla/pml_yalla.c @@ -265,6 +265,7 @@ int mca_pml_yalla_add_procs(struct ompi_proc_t **procs, size_t nprocs) int mca_pml_yalla_del_procs(struct ompi_proc_t **procs, size_t nprocs) { size_t i; + int ret; if (ompi_mpi_state >= OMPI_MPI_STATE_FINALIZE_STARTED) { PML_YALLA_VERBOSE(3, "%s", "using bulk powerdown"); @@ -276,7 +277,9 @@ int mca_pml_yalla_del_procs(struct ompi_proc_t **procs, size_t nprocs) PML_YALLA_VERBOSE(2, "disconnected from rank %s", OPAL_NAME_PRINT(procs[i]->super.proc_name)); procs[i]->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_PML] = NULL; } - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + return ret; + } return OMPI_SUCCESS; } diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c index 538e7fe2ad5..7c326da8005 100644 --- a/ompi/runtime/ompi_mpi_finalize.c +++ b/ompi/runtime/ompi_mpi_finalize.c @@ -254,7 +254,13 @@ int ompi_mpi_finalize(void) * communications/actions to complete. See * https://github.com/open-mpi/ompi/issues/1576 for the * original bug report. */ - opal_pmix.fence_nb(NULL, 0, fence_cbfunc, (void*)&active); + if (OMPI_SUCCESS != (ret = opal_pmix.fence_nb(NULL, 0, fence_cbfunc, + (void*)&active))) { + OMPI_ERROR_LOG(ret); + /* Reset the active flag to false, to avoid waiting for + * completion when the fence was failed. */ + active = false; + } OMPI_LAZY_WAIT_FOR_COMPLETION(active); } else { /* However, we cannot guarantee that the provided PMIx has @@ -265,7 +271,9 @@ int ompi_mpi_finalize(void) ompi_communicator_t *comm = &ompi_mpi_comm_world.comm; comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module); - opal_pmix.fence(NULL, 0); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + OMPI_ERROR_LOG(ret); + } } } diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 97ece1dae1e..f824a454116 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -678,18 +678,31 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided, * at a time, so we will need to wait when we * get there */ active = true; - opal_pmix.fence_nb(NULL, true, fence_release, (void*)&active); + ret = opal_pmix.fence_nb(NULL, true, fence_release, + (void*)&active); } else { - opal_pmix.fence_nb(NULL, true, NULL, NULL); + ret = opal_pmix.fence_nb(NULL, true, NULL, NULL); + } + if (OMPI_SUCCESS != ret) { + error = "opal_pmix.fence_nb() failed"; + goto error; } } else if (!opal_pmix_base_async_modex) { active = true; - opal_pmix.fence_nb(NULL, opal_pmix_collect_all_data, - fence_release, (void*)&active); + if (OMPI_SUCCESS != (ret = opal_pmix.fence_nb(NULL, + opal_pmix_collect_all_data, fence_release, + (void*)&active))) { + error = "opal_pmix.fence_nb() failed"; + goto error; + } OMPI_LAZY_WAIT_FOR_COMPLETION(active); } } else { - opal_pmix.fence(NULL, opal_pmix_collect_all_data); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, + opal_pmix_collect_all_data))) { + error = "opal_pmix.fence() failed"; + goto error; + } } OMPI_TIMING_NEXT("modex"); @@ -867,11 +880,17 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided, } else { active = true; if (NULL != opal_pmix.fence_nb) { - opal_pmix.fence_nb(NULL, false, - fence_release, (void*)&active); + if (OMPI_SUCCESS != (ret = opal_pmix.fence_nb(NULL, false, + fence_release, (void*)&active))) { + error = "opal_pmix.fence_nb() failed"; + goto error; + } OMPI_LAZY_WAIT_FOR_COMPLETION(active); } else { - opal_pmix.fence(NULL, false); + if (OMPI_SUCCESS != (ret = opal_pmix.fence(NULL, false))) { + error = "opal_pmix.fence() failed"; + goto error; + } } } } diff --git a/orte/mca/ess/pmi/ess_pmi_module.c b/orte/mca/ess/pmi/ess_pmi_module.c index 0420218e26b..8e8f90019be 100644 --- a/orte/mca/ess/pmi/ess_pmi_module.c +++ b/orte/mca/ess/pmi/ess_pmi_module.c @@ -441,7 +441,10 @@ static int rte_init(void) if (ORTE_PROC_IS_NON_MPI && !orte_do_not_barrier) { /* need to commit the data before we fence */ opal_pmix.commit(); - opal_pmix.fence(NULL, 0); + if (ORTE_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + error = "opal_pmix.fence() failed"; + goto error; + } } return ORTE_SUCCESS; diff --git a/orte/mca/snapc/full/snapc_full_app.c b/orte/mca/snapc/full/snapc_full_app.c index 1dd5a8d5edd..3a436bc81d4 100644 --- a/orte/mca/snapc/full/snapc_full_app.c +++ b/orte/mca/snapc/full/snapc_full_app.c @@ -150,7 +150,11 @@ int app_coord_init() "app) Startup Barrier...")); } - opal_pmix.fence(NULL, 0); + if (ORTE_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + ORTE_ERROR_LOG(ret); + exit_status = ret; + goto cleanup; + } if( 0 == ORTE_PROC_MY_NAME->vpid ) { OPAL_OUTPUT_VERBOSE((3, mca_snapc_full_component.super.output_handle, @@ -216,7 +220,11 @@ int app_coord_finalize() "app) Shutdown Barrier...")); } - opal_pmix.fence(NULL, 0); + if (ORTE_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + ORTE_ERROR_LOG(ret); + exit_status = ret; + goto cleanup; + } if( 0 == ORTE_PROC_MY_NAME->vpid ) { OPAL_OUTPUT_VERBOSE((3, mca_snapc_full_component.super.output_handle,