init/finalize: extensions

ompi/mpi/c/finalized.c

@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +42,14 @@ int MPI_Finalized(int *flag)
 
     OPAL_CR_NOOP_PROGRESS();
 
+    /* We must obtain the lock to guarnatee consistent values of
+       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
+       this lock is held for the bulk of the duration of
+       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
+       lock, we are guaranteed that some other thread is not part way
+       through initialization or finalization. */
+    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
 
@@ -49,17 +58,19 @@ int MPI_Finalized(int *flag)
                MPI_Finalize) or not */
 
             if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG,
                                               FUNC_NAME);
             }
         }
     }
 
-    /* Pretty simple */
-
     *flag = ompi_mpi_finalized;
+    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+
     return MPI_SUCCESS;
 }

ompi/mpi/c/init.c

@@ -9,7 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -47,25 +47,6 @@ int MPI_Init(int *argc, char ***argv)
     char *env;
     int required = MPI_THREAD_SINGLE;
 
-    /* Ensure that we were not already initialized or finalized */
-
-    if (ompi_mpi_finalized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt",
-                           "mpi-function-after-finalize", true, FUNC_NAME);
-        }
-        return ompi_errhandler_invoke(NULL, NULL,
-                                      OMPI_ERRHANDLER_TYPE_COMM,
-                                      MPI_ERR_OTHER, FUNC_NAME);
-    } else if (ompi_mpi_initialized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-initialize-twice",
-                           true, FUNC_NAME);
-        }
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER,
-                                      FUNC_NAME);
-    }
-
     /* check for environment overrides for required thread level.  If
        there is, check to see that it is a valid/supported thread level.
        If not, default to MPI_THREAD_MULTIPLE. */

ompi/mpi/c/init_thread.c

@@ -12,6 +12,7 @@
  * Copyright (c) 2010      Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -60,23 +61,6 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
     *provided = MPI_THREAD_SINGLE;
 #endif
 
-    /* Ensure that we were not already initialized or finalized */
-
-    if (ompi_mpi_finalized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-function-after-finalize",
-                           true, FUNC_NAME);
-        }
-        return ompi_errhandler_invoke(NULL, NULL, OMPI_ERRHANDLER_TYPE_COMM,
-                                      MPI_ERR_OTHER, FUNC_NAME);
-    } else if (ompi_mpi_initialized) {
-        if (0 == ompi_comm_rank(MPI_COMM_WORLD)) {
-            opal_show_help("help-mpi-api.txt", "mpi-initialize-twice",
-                           true, FUNC_NAME);
-        }
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, FUNC_NAME);
-    }
-
     /* Call the back-end initialization function (we need to put as
        little in this function as possible so that if it's profiled, we
        don't lose anything) */

ompi/mpi/c/initialized.c

@@ -11,6 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +42,14 @@ int MPI_Initialized(int *flag)
 
     OPAL_CR_NOOP_PROGRESS();
 
+    /* We must obtain the lock to guarnatee consistent values of
+       ompi_mpi_initialized and ompi_mpi_finalized.  Note, too, that
+       this lock is held for the bulk of the duration of
+       ompi_mpi_init() and ompi_mpi_finalize(), so when we get the
+       lock, we are guaranteed that some other thread is not part way
+       through initialization or finalization. */
+    opal_mutex_lock(&ompi_mpi_bootstrap_mutex);
+
     if (MPI_PARAM_CHECK) {
         if (NULL == flag) {
 
@@ -49,17 +58,19 @@ int MPI_Initialized(int *flag)
                MPI_Finalize) or not */
 
             if (ompi_mpi_initialized && !ompi_mpi_finalized) {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_ARG,
                                               FUNC_NAME);
             } else {
+                opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
                 return OMPI_ERRHANDLER_INVOKE(null, MPI_ERR_ARG,
                                               FUNC_NAME);
             }
         }
     }
 
-    /* Pretty simple */
-
     *flag = ompi_mpi_initialized;
+    opal_mutex_unlock(&ompi_mpi_bootstrap_mutex);
+
     return MPI_SUCCESS;
 }

ompi/mpi/help-mpi-api.txt

@@ -11,15 +11,6 @@
 #
 # This is the US/English general help file for Open MPI.
 #
-[mpi-function-after-finalize]
-Calling most MPI functions after calling MPI_Finalize is erroneous.
-
-There are a small number of exceptions, such as MPI_Initialized,
-MPI_Finalized, and MPI_Get_version.
-#
-[mpi-initialize-twice]
-Calling MPI_Init or MPI_Init_thread twice is erroneous.
-#
 [mpi-abort]
 MPI_ABORT was invoked on rank %d in communicator %s
 with errorcode %d.

ompi/mpi/man/man3/MPI_Finalize.3in

@@ -1,5 +1,5 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
@@ -63,3 +63,10 @@ Before the error value is returned, the current MPI error handler is
 called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
 
 
+.SH SEE ALSO
+.ft R
+.nf
+MPI_Init
+MPI_Init_thread
+MPI_Initialized
+MPI_Finalized

ompi/mpi/man/man3/MPI_Finalized.3in

@@ -1,11 +1,11 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
 .TH MPI_Finalized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .SH NAME
-\fBMPI_Finalized \fP \- Checks whether MPI_Finalize has completed.
+\fBMPI_Finalized \fP \- Checks whether MPI has been finalized
 
 .SH SYNTAX
 .ft R
@@ -42,19 +42,29 @@ bool MPI::Is_finalized()
 .ft R
 .TP 1i
 flag
-True if MPI was finalized (logical).
+True if MPI was finalized, and false otherwise (logical).
 .ft R
 .TP 1i
 IERROR
 Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine returns "true" if MPI_Finalize has completed. You can call MPI_Finalized before MPI_Init and after MPI_Finalize.
+This routine may be used to determine whether MPI has been finalized.
+It is one of a small number of routines that may be called before MPI
+is initialized and after MPI has been finalized (MPI_Initialized is
+another).
 
 .SH ERRORS
 Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object.
 .sp
 Before the error value is returned, the current MPI error handler is
 called. By default, this error handler aborts the MPI job, except for I/O function errors. The error handler may be changed with MPI_Comm_set_errhandler; the predefined error handler MPI_ERRORS_RETURN may be used to cause error values to be returned. Note that MPI does not guarantee that an MPI program can continue past an error.
 
+.SH SEE ALSO
+.ft R
+.nf
+MPI_Init
+MPI_Init_thread
+MPI_Initialized
+MPI_Finalize

ompi/mpi/man/man3/MPI_Init.3in

@@ -1,5 +1,5 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
@@ -56,10 +56,10 @@ Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine, or MPI_Init_thread, must be called before any other MPI
-routine (apart from MPI_Initialized) is called. MPI can be initialized
-at most once; subsequent calls to MPI_Init or MPI_Init_thread are
-erroneous.
+This routine, or MPI_Init_thread, must be called before most other MPI
+routines are called.  There are a small number of exceptions, such as
+MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
+once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
 .sp
 All MPI programs must contain a call to MPI_Init or
 MPI_Init_thread. Open MPI accepts the C/C++ \fIargc\fP and \fIargv\fP
@@ -110,4 +110,4 @@ See the MPI man page for a full list of MPI error codes.
 MPI_Init_thread
 MPI_Initialized
 MPI_Finalize
-
+MPI_Finalized

ompi/mpi/man/man3/MPI_Init_thread.3in

@@ -1,7 +1,7 @@
 .\" -*- nroff -*-
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
-.\" Copyright (c) 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" $COPYRIGHT$
 .TH MPI_Init_thread 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .
@@ -67,8 +67,9 @@ Fortran only: Error status (integer).
 .
 .SH DESCRIPTION
 .ft R
-This routine, or MPI_Init, must be called before any other MPI routine
-(apart from MPI_Initialized) is called. MPI can be initialized at most
+This routine, or MPI_Init, must be called before most other MPI
+routines are called. There are a small number of exceptions, such as
+MPI_Initialized and MPI_Finalized.  MPI can be initialized at most
 once; subsequent calls to MPI_Init or MPI_Init_thread are erroneous.
 .sp
 MPI_Init_thread, as compared to MPI_Init, has a provision to request a
@@ -203,4 +204,4 @@ guarantee that an MPI program can continue past an error.
 MPI_Init
 MPI_Initialized
 MPI_Finalize
-
+MPI_Finalized

ompi/mpi/man/man3/MPI_Initialized.3in

@@ -1,11 +1,11 @@
 .\" -*- nroff -*-
-.\" Copyright 2010 Cisco Systems, Inc.  All rights reserved.
+.\" Copyright (c) 2010-2015 Cisco Systems, Inc.  All rights reserved.
 .\" Copyright 2006-2008 Sun Microsystems, Inc.
 .\" Copyright (c) 1996 Thinking Machines Corporation
 .\" $COPYRIGHT$
 .TH MPI_Initialized 3 "#OMPI_DATE#" "#PACKAGE_VERSION#" "#PACKAGE_NAME#"
 .SH NAME
-\fBMPI_Initialized\fP \- Indicates whether MPI_Init has been called.
+\fBMPI_Initialized\fP \- Checks whether MPI has been initialized
 
 .SH SYNTAX
 .ft R
@@ -42,15 +42,18 @@ bool Is_initialized()
 .ft R
 .TP 1i
 flag
-Flag is true if MPI_Init has been called and false otherwise.
+True if MPI has been initialized, and false otherwise (logical).
 .ft R
 .TP 1i
 IERROR
 Fortran only: Error status (integer).
 
 .SH DESCRIPTION
 .ft R
-This routine may be used to determine whether MPI_Init has been called. It is the only routine that may be called before MPI_Init is called.
+This routine may be used to determine whether MPI has been
+initialized.  It is one of a small number of routines that may be
+called before MPI is initialized and after MPI has been finalized
+(MPI_Finalized is another).
 
 .SH ERRORS
 Almost all MPI routines return an error value; C routines as the value of the function and Fortran routines in the last argument. C++ functions do not return errors. If the default error handler is set to MPI::ERRORS_THROW_EXCEPTIONS, then on error the C++ exception mechanism will be used to throw an MPI::Exception object.
@@ -60,6 +63,8 @@ called. By default, this error handler aborts the MPI job, except for I/O functi
 
 .SH SEE ALSO
 .ft R
-.sp
+.nf
 MPI_Init
-
+MPI_Init_thread
+MPI_Finalize
+MPI_Finalized

ompi/runtime/help-mpi-runtime.txt

@@ -50,6 +50,24 @@ You may wish to try to narrow down the problem;
 WARNING: The MCA parameter mpi_param_check has been set to true, but
 parameter checking has been compiled out of Open MPI.  The
 mpi_param_check value has therefore been ignored.
+#
+[mpi_init: invoked multiple times]
+Open MPI has detected that this process has attempted to initialize
+MPI (via MPI_INIT or MPI_INIT_THREAD) more than once.  This is
+erroneous.
+#
+[mpi_init: already finalized]
+Open MPI has detected that this process has attempted to initialize
+MPI (via MPI_INIT or MPI_INIT_THREAD) after MPI_FINALIZE has been
+called.  This is erroneous.
+#
+[mpi_finalize: not initialized]
+The function MPI_FINALIZE was invoked before MPI was initialized in a
+process on host %s, PID %d.
+
+This indicates an erroneous MPI program; MPI must be initialized
+before it can be finalized.
+#
 [mpi_finalize:invoked_multiple_times]
 The function MPI_FINALIZE was invoked multiple times in a single
 process on host %s, PID %d.

ompi/runtime/mpiruntime.h

@@ -35,6 +35,7 @@
 
 #include "opal/class/opal_list.h"
 #include "opal/class/opal_hash_table.h"
+#include "opal/threads/mutex.h"
 
 BEGIN_C_DECLS
 
@@ -47,16 +48,18 @@ struct ompi_predefined_datatype_t;
 
 /* Global variables and symbols for the MPI layer */
 
-/** Did mpi start to initialize? */
-OMPI_DECLSPEC extern bool ompi_mpi_init_started;
-/** Is mpi initialized? */
-OMPI_DECLSPEC extern bool ompi_mpi_initialized;
-/** Has mpi been finalized? */
-OMPI_DECLSPEC extern bool ompi_mpi_finalized;
+/** Mutex to protect all the _init and _finalize variables */
+OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex;
+/** Did MPI start to initialize? */
+OMPI_DECLSPEC extern volatile bool ompi_mpi_init_started;
 /** Has the RTE been initialized? */
-OMPI_DECLSPEC extern bool ompi_rte_initialized;
-/** Did mpi start to finalize? */
-OMPI_DECLSPEC extern int32_t ompi_mpi_finalize_started;
+OMPI_DECLSPEC extern volatile bool ompi_rte_initialized;
+/** Is MPI fully initialized? */
+OMPI_DECLSPEC extern volatile bool ompi_mpi_initialized;
+/** Did MPI start to finalize? */
+OMPI_DECLSPEC extern volatile bool ompi_mpi_finalize_started;
+/** Has MPI been fully finalized? */
+OMPI_DECLSPEC extern volatile bool ompi_mpi_finalized;
 
 /** Do we have multiple threads? */
 OMPI_DECLSPEC extern bool ompi_mpi_thread_multiple;