Restore the coll/sync module and provide a test to verify its operation

.gitignore

@@ -395,6 +395,7 @@ orte/test/mpi/pconnect
 orte/test/mpi/thread_init
 orte/test/mpi/memcached-dummy
 orte/test/mpi/coll_test
+orte/test/mpi/badcoll
 
 orte/test/system/radix
 orte/test/system/sigusr_trap

ompi/mca/coll/sync/Makefile.am

@@ -0,0 +1,52 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2016      Intel, Inc. All rights reserved
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+dist_ompidata_DATA = help-coll-sync.txt
+
+sources = \
+        coll_sync.h \
+        coll_sync_component.c \
+        coll_sync_module.c \
+        coll_sync_bcast.c \
+        coll_sync_exscan.c \
+        coll_sync_gather.c \
+        coll_sync_gatherv.c \
+        coll_sync_reduce.c \
+        coll_sync_reduce_scatter.c \
+        coll_sync_scan.c \
+        coll_sync_scatter.c \
+        coll_sync_scatterv.c
+
+if MCA_BUILD_ompi_coll_sync_DSO
+component_noinst =
+component_install = mca_coll_sync.la
+else
+component_noinst = libmca_coll_sync.la
+component_install =
+endif
+
+mcacomponentdir = $(ompilibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_sync_la_SOURCES = $(sources)
+mca_coll_sync_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_sync_la_SOURCES =$(sources)
+libmca_coll_sync_la_LDFLAGS = -module -avoid-version

ompi/mca/coll/sync/coll_sync.h

@@ -0,0 +1,183 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_SYNC_EXPORT_H
+#define MCA_COLL_SYNC_EXPORT_H
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+
+#include "opal/class/opal_object.h"
+#include "opal/mca/mca.h"
+#include "opal/util/output.h"
+
+#include "ompi/constants.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/communicator/communicator.h"
+
+BEGIN_C_DECLS
+
+/* API functions */
+
+int mca_coll_sync_init_query(bool enable_progress_threads,
+                             bool enable_mpi_threads);
+mca_coll_base_module_t
+*mca_coll_sync_comm_query(struct ompi_communicator_t *comm,
+                          int *priority);
+
+int mca_coll_sync_module_enable(mca_coll_base_module_t *module,
+                                struct ompi_communicator_t *comm);
+
+int mca_coll_sync_barrier(struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_sync_bcast(void *buff, int count,
+                        struct ompi_datatype_t *datatype,
+                        int root,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module);
+
+int mca_coll_sync_exscan(const void *sbuf, void *rbuf, int count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_sync_gather(const void *sbuf, int scount,
+                         struct ompi_datatype_t *sdtype,
+                         void *rbuf, int rcount,
+                         struct ompi_datatype_t *rdtype,
+                         int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_sync_gatherv(const void *sbuf, int scount,
+                          struct ompi_datatype_t *sdtype,
+                          void *rbuf, int *rcounts, int *disps,
+                          struct ompi_datatype_t *rdtype,
+                          int root,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_sync_reduce(const void *sbuf, void *rbuf, int count,
+                         struct ompi_datatype_t *dtype,
+                         struct ompi_op_t *op,
+                         int root,
+                         struct ompi_communicator_t *comm,
+                         mca_coll_base_module_t *module);
+
+int mca_coll_sync_reduce_scatter(const void *sbuf, void *rbuf,
+                                 int *rcounts,
+                                 struct ompi_datatype_t *dtype,
+                                 struct ompi_op_t *op,
+                                 struct ompi_communicator_t *comm,
+                                 mca_coll_base_module_t *module);
+
+int mca_coll_sync_scan(const void *sbuf, void *rbuf, int count,
+                       struct ompi_datatype_t *dtype,
+                       struct ompi_op_t *op,
+                       struct ompi_communicator_t *comm,
+                       mca_coll_base_module_t *module);
+
+int mca_coll_sync_scatter(const void *sbuf, int scount,
+                          struct ompi_datatype_t *sdtype,
+                          void *rbuf, int rcount,
+                          struct ompi_datatype_t *rdtype,
+                          int root,
+                          struct ompi_communicator_t *comm,
+                          mca_coll_base_module_t *module);
+
+int mca_coll_sync_scatterv(const void *sbuf, int *scounts, int *disps,
+                           struct ompi_datatype_t *sdtype,
+                           void *rbuf, int rcount,
+                           struct ompi_datatype_t *rdtype,
+                           int root,
+                           struct ompi_communicator_t *comm,
+                           mca_coll_base_module_t *module);
+
+
+/* Types */
+/* Module */
+
+typedef struct mca_coll_sync_module_t {
+    mca_coll_base_module_t super;
+
+    /* Pointers to all the "real" collective functions */
+    mca_coll_base_comm_coll_t c_coll;
+
+    /* How many ops we've executed */
+    int before_num_operations;
+
+    /* How many ops we've executed (it's easier to have 2) */
+    int after_num_operations;
+
+    /* Avoid recursion of syncs */
+    bool in_operation;
+} mca_coll_sync_module_t;
+
+OBJ_CLASS_DECLARATION(mca_coll_sync_module_t);
+
+/* Component */
+
+typedef struct mca_coll_sync_component_t {
+    mca_coll_base_component_2_0_0_t super;
+
+    /* Priority of this component */
+    int priority;
+
+    /* Do a sync *before* each Nth collective */
+    int barrier_before_nops;
+
+    /* Do a sync *after* each Nth collective */
+    int barrier_after_nops;
+} mca_coll_sync_component_t;
+
+/* Globally exported variables */
+
+OMPI_MODULE_DECLSPEC extern mca_coll_sync_component_t mca_coll_sync_component;
+
+/* Macro used in most of the collectives */
+
+#define COLL_SYNC(m, op) \
+do { \
+    int err = MPI_SUCCESS; \
+    (m)->in_operation = true; \
+    if (OPAL_UNLIKELY(++((m)->before_num_operations) ==                         \
+                      mca_coll_sync_component.barrier_before_nops)) {           \
+        (m)->before_num_operations = 0;                                         \
+        err = (m)->c_coll.coll_barrier(comm, (m)->c_coll.coll_barrier_module);  \
+    }                                                                           \
+    if (OPAL_LIKELY(MPI_SUCCESS == err)) {                                      \
+        err = op;                                                               \
+    }                                                                           \
+    if (OPAL_UNLIKELY(++((m)->after_num_operations) ==                          \
+                      mca_coll_sync_component.barrier_after_nops) &&            \
+        OPAL_LIKELY(MPI_SUCCESS == err)) {                                      \
+        (m)->after_num_operations = 0;                                          \
+        err = (m)->c_coll.coll_barrier(comm, (m)->c_coll.coll_barrier_module);  \
+    }                                                                           \
+    (m)->in_operation = false;                                                  \
+    return err;                                                                 \
+} while(0)
+
+END_C_DECLS
+
+#endif /* MCA_COLL_SYNC_EXPORT_H */

ompi/mca/coll/sync/coll_sync_bcast.c

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2009      Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "coll_sync.h"
+
+
+/*
+ *	bcast
+ *
+ *	Function:	- broadcast
+ *	Accepts:	- same arguments as MPI_Bcast()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int mca_coll_sync_bcast(void *buff, int count,
+                        struct ompi_datatype_t *datatype, int root,
+                        struct ompi_communicator_t *comm,
+                        mca_coll_base_module_t *module)
+{
+    mca_coll_sync_module_t *s = (mca_coll_sync_module_t*) module;
+
+    if (s->in_operation) {
+        return s->c_coll.coll_bcast(buff, count, datatype, root, comm,
+                                    s->c_coll.coll_bcast_module);
+    } else {
+        COLL_SYNC(s, s->c_coll.coll_bcast(buff, count, datatype, root, comm,
+                                          s->c_coll.coll_bcast_module));
+    }
+}

ompi/mca/coll/sync/coll_sync_component.c

@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2008-2009 Cisco Systems, Inc.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "opal/util/output.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "coll_sync.h"
+
+/*
+ * Public string showing the coll ompi_sync component version number
+ */
+const char *mca_coll_sync_component_version_string =
+    "Open MPI sync collective MCA component version " OMPI_VERSION;
+
+/*
+ * Local function
+ */
+static int sync_register(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+mca_coll_sync_component_t mca_coll_sync_component = {
+    {
+        /* First, the mca_component_t struct containing meta information
+         * about the component itself */
+
+       .collm_version = {
+            MCA_COLL_BASE_VERSION_2_0_0,
+
+            /* Component name and version */
+            .mca_component_name = "sync",
+            MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
+                                  OMPI_RELEASE_VERSION),
+
+            /* Component open and close functions */
+            .mca_register_component_params = sync_register
+        },
+        .collm_data = {
+            /* The component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
+
+        /* Initialization / querying functions */
+
+        .collm_init_query = mca_coll_sync_init_query,
+        .collm_comm_query = mca_coll_sync_comm_query
+    },
+};
+
+
+static int sync_register(void)
+{
+    mca_base_component_t *c = &mca_coll_sync_component.super.collm_version;
+
+    mca_coll_sync_component.priority = 50;
+    (void) mca_base_component_var_register(c, "priority",
+                                           "Priority of the sync coll component; only relevant if barrier_before or barrier_after is > 0",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_sync_component.priority);
+
+    mca_coll_sync_component.barrier_before_nops = 0;
+    (void) mca_base_component_var_register(c, "barrier_before",
+                                           "Do a synchronization before each Nth collective",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_sync_component.barrier_before_nops);
+
+    mca_coll_sync_component.barrier_after_nops = 0;
+    (void) mca_base_component_var_register(c, "barrier_after",
+                                           "Do a synchronization after each Nth collective",
+                                           MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                           OPAL_INFO_LVL_9,
+                                           MCA_BASE_VAR_SCOPE_READONLY,
+                                           &mca_coll_sync_component.barrier_after_nops);
+
+    return OMPI_SUCCESS;
+}