Add a new topo module: Treematch

README

@@ -1,7 +1,7 @@
 Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
                         University Research and Technology
                         Corporation.  All rights reserved.
-Copyright (c) 2004-2007 The University of Tennessee and The University
+Copyright (c) 2004-2015 The University of Tennessee and The University
                         of Tennessee Research Foundation.  All rights
                         reserved.
 Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -436,6 +436,9 @@ General Run-Time Support Notes
 MPI Functionality and Features
 ------------------------------
 
+- Rank reordering support is available using the TreeMatch library. It is activated
+  for the graph and dist_graph topologies.
+
 - All MPI-3 functionality is supported.
 
 - When using MPI deprecated functions, some compilers will emit

ompi/mca/topo/treematch/Makefile.am

@@ -0,0 +1,62 @@
+#
+# Copyright (c) 2011-2015 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2011-2015 INRIA.  All rights reserved.
+# Copyright (c) 2011-2015 Université Bordeaux 1
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+if topo_treematch_local
+extra_treematch_files = treematch/tm_bucket.h \
+    treematch/tm_hwloc.h treematch/tm_mapping.h \
+    treematch/tm_timings.h treematch/tm_tree.h \
+    treematch/tm_kpartitioning.h treematch/uthash.h\
+    treematch/IntConstantInitializedVector.h \
+    treematch/tm_mt.h \
+    treematch/tm_thread_pool.h treematch/tm_verbose.h \
+    treematch/tm_malloc.h \
+    treematch/IntConstantInitializedVector.c \
+    treematch/tm_mt.c \
+    treematch/tm_thread_pool.c treematch/tm_verbose.c \
+    treematch/tm_malloc.c \
+    treematch/tm_mapping.c treematch/tm_timings.c \
+    treematch/tm_bucket.c treematch/tm_tree.c \
+    treematch/tm_hwloc.c treematch/tm_kpartitioning.c
+endif
+
+sources = \
+    topo_treematch.h \
+    topo_treematch_module.c \
+    topo_treematch_component.c \
+    topo_treematch_dist_graph_create.c $(extra_treematch_files)
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_ompi_topo_treematch_DSO
+lib =
+lib_sources =
+component = mca_topo_treematch.la
+component_sources = $(sources)
+else
+lib = libmca_topo_treematch.la
+lib_sources = $(sources)
+component =
+component_sources =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component)
+mca_topo_treematch_la_SOURCES = $(component_sources)
+mca_topo_treematch_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(lib)
+libmca_topo_treematch_la_SOURCES = $(lib_sources)
+libmca_topo_treematch_la_LDFLAGS = -module -avoid-version
+

ompi/mca/topo/treematch/configure.m4

@@ -0,0 +1,87 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2011-2015 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2011-2015 INRIA.  All rights reserved.
+# Copyright (c) 2011-2015 Universite Bordeaux 1
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# MCA_ompi_topo_treematch_CONFIG([action-if-can-compile],
+#                                [action-if-cant-compile])
+# -------------------------------------------
+AC_DEFUN([MCA_ompi_topo_treematch_CONFIG], [
+    AC_REQUIRE([MCA_opal_hwloc_CONFIG_REQUIRE])
+
+    AC_ARG_WITH([treematch],
+          [AC_HELP_STRING([--with-treematch(=DIR)],
+               [Build TreeMatch topology support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])],
+               [],
+               [with_treematch=yes])
+     AC_ARG_WITH([treematch-include],
+                 [AC_HELP_STRING([--with-treematch-include(=DIR)],
+                                 ["Search for TreeMatch headers in DIR"])])
+     AC_ARG_WITH([treematch-libdir],
+                 [AC_HELP_STRING([--with-treematch-libdir(=DIR)],
+                                 ["Search for TreeMatch libraries in DIR"])])
+
+    treematch_files_local="no"
+    ompi_check_treematch_dir=$srcdir
+    ompi_check_treematch_libdir=""
+    ompi_check_treematch_happy="no"
+
+    AS_IF([test "x$with_treematch" != xno],
+          [AC_MSG_CHECKING([TreeMatch headers])
+           AS_IF([test "x$with_treematch_include" = x],
+                 [AS_IF([test "x$with_treematch" = xyes],
+                        [treematch_files_local="yes"
+                         with_treematch_include=$OMPI_TOP_SRCDIR/ompi/mca/topo/treematch/treematch],
+                        [with_treematch_include=$with_treematch/include])])
+           AS_IF([test -f $with_treematch_include/tm_tree.h],
+                 [AS_IF([test "x$with_treematch" = xyes],
+                        [AC_MSG_RESULT([in the source])],
+                        [AC_MSG_RESULT([user provided])])
+                  opal_check_treematch_dir=$with_treematch_include
+                  ompi_check_treematch_happy="yes"],
+                 [AC_MSG_ERROR([missing tm_tree.h (${with_treematch}:${with_treematch_include})])])])
+
+    AS_IF([test "$ompi_check_treematch_happy" = "yes"],
+          [AC_MSG_CHECKING([TreeMatch library])
+           OPAL_CHECK_WITHDIR([treematch], [$with_treematch_include], [tm_tree.h])
+           AS_IF([test "x$with_treematch_libdir" = x],
+                 [AS_IF([test "x$with_treematch" != xyes],
+                        [with_treematch_libdir=$with_treematch/lib]
+                        [with_treematch_libdir=$OMPI_TOP_SRCDIR/ompi/mca/topo/treematch/treematch])])
+           AS_IF([test "x$treematch_files_local" = xno],
+                 [OPAL_CHECK_WITHDIR([treematch-libdir], [$with_treematch_libdir], [libtreematch.*])
+                  AS_IF([test "x$with_treematch" != xno -a "x$with_treematch" != xyes],
+                        [AS_IF([test ! -z "$with_treematch" -a "$with_treematch" != "yes"],
+                               [ompi_check_treematch_dir="$with_treematch"])
+                         AS_IF([test ! -z "$with_treematch_libdir" -a "$with_treematch_libdir" != "yes"],
+                               [ompi_check_treematch_libdir="$with_treematch_libdir"])
+                         OPAL_CHECK_PACKAGE([topo_treematch],
+                                            [tm_tree.h],
+                                            [treematch],
+                                            [build_tree],
+                                            [],
+                                            [$with_treematch_include],
+                                            [$with_treematch_libdir],
+                                            [ompi_check_treematch_happy="yes"],
+                                            [ompi_check_treematch_happy="no"])],
+                        [ompi_check_treematch_happy="no"])])])
+
+    AS_IF([test "$ompi_check_treematch_happy" = "yes"],
+          [$1],
+          [AS_IF([test ! -z "$with_treematch" -a "$with_treematch" != "no"],
+                 [AC_MSG_ERROR([TreeMatch support requested but not found.  Aborting])])
+           $2])
+
+    AC_CONFIG_FILES([ompi/mca/topo/treematch/Makefile])
+    AM_CONDITIONAL(topo_treematch_local,
+                   [test "x$treematch_files_local" = "xyes"])
+])

ompi/mca/topo/treematch/topo_treematch.h

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2011-2015 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2011-2015 INRIA.  All rights reserved.
+ * Copyright (c) 2011-2015 Bordeaux Polytechnic Institute
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_TOPO_UNTIY_H
+#define MCA_TOPO_UNTIY_H
+
+#include "ompi_config.h"
+#include "ompi/mca/topo/topo.h"
+
+/*
+ * ******************************************************************
+ * ******** functions which provide MCA interface comppliance *******
+ * ******************************************************************
+ * These functions are:
+ *       - mca_topo_treematch_module_open
+ *       - mca_topo_treematch_module_close
+ *       - mca_topo_treematch_module_query
+ *       - mca_topo_treematch_module_finalize
+ * These functions are always found on the mca_topo_treematch_module
+ * structure. They are the "meta" functions to ensure smooth op.
+ * ******************************************************************
+ */
+BEGIN_C_DECLS
+
+/*
+ * Public component instance
+ */
+typedef struct mca_topo_treematch_component_2_2_0_t {
+    mca_topo_base_component_2_2_0_t super;
+
+    int reorder_mode;
+} mca_topo_treematch_component_2_2_0_t;
+
+OMPI_MODULE_DECLSPEC extern mca_topo_treematch_component_2_2_0_t
+    mca_topo_treematch_component;
+
+/*
+ * A unique module class for the module so that we can both cache
+ * module-specific information on the module and have a
+ * module-specific constructor and destructor.
+ */
+typedef struct {
+    mca_topo_base_module_t super;
+
+    /* Modules can add their own information here */
+} mca_topo_treematch_module_t;
+
+OBJ_CLASS_DECLARATION(mca_topo_treematch_module_t);
+
+
+/*
+ * Module functions
+ */
+
+int mca_topo_treematch_dist_graph_create(mca_topo_base_module_t* module,
+                                         ompi_communicator_t *comm_old,
+                                         int n, int nodes[],
+                                         int degrees[], int targets[],
+                                         int weights[],
+                                         struct ompi_info_t *info, int reorder,
+                                         ompi_communicator_t **newcomm);
+/*
+ * ******************************************************************
+ * ************ functions implemented in this module end ************
+ * ******************************************************************
+ */
+
+END_C_DECLS
+
+#endif /* MCA_TOPO_EXAMPLE_H */

ompi/mca/topo/treematch/topo_treematch_component.c

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2011-2015 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2011-2015 INRIA.  All rights reserved.
+ * Copyright (c) 2011-2015 Université Bordeaux 1
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "ompi/mca/topo/treematch/topo_treematch.h"
+
+/*
+ * Public string showing the topo treematch module version number
+ */
+const char *mca_topo_treematch_component_version_string =
+    "Open MPI treematch topology MCA component version" OMPI_VERSION;
+
+/*
+ * Local funtions
+ */
+static int init_query(bool enable_progress_threads, bool enable_mpi_threads);
+static struct mca_topo_base_module_t *
+comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type);
+static int mca_topo_treematch_component_register(void);
+
+/*
+ * Public component structure
+ */
+mca_topo_treematch_component_2_2_0_t mca_topo_treematch_component =
+    {
+        {
+            {
+                MCA_TOPO_BASE_VERSION_2_2_0,
+
+                "treematch",
+                OMPI_MAJOR_VERSION,
+                OMPI_MINOR_VERSION,
+                OMPI_RELEASE_VERSION,
+                NULL,  /* component open */
+                NULL,  /* component close */
+                NULL, /* component query */
+                mca_topo_treematch_component_register, /* component register */
+            },
+
+            {
+                /* The component is checkpoint ready */
+                MCA_BASE_METADATA_PARAM_CHECKPOINT
+            },
+
+            init_query,
+            comm_query
+        },
+        0  /* reorder: by default centralized */
+    };
+
+
+static int init_query(bool enable_progress_threads, bool enable_mpi_threads)
+{
+    if(NULL == opal_hwloc_topology) {
+        return OPAL_ERR_NOT_SUPPORTED;
+    }
+    return OMPI_SUCCESS;
+}
+
+
+static struct mca_topo_base_module_t *
+comm_query(const ompi_communicator_t *comm, int *priority, uint32_t type)
+{
+    mca_topo_treematch_module_t *treematch;
+
+    if( OMPI_COMM_DIST_GRAPH != type ) {
+        return NULL;
+    }
+    treematch = OBJ_NEW(mca_topo_treematch_module_t);
+    if (NULL == treematch) {
+        return NULL;
+    }
+    treematch->super.topo.dist_graph.dist_graph_create = mca_topo_treematch_dist_graph_create;
+
+    /* This component has very low priority -- it's an treematch, after
+       all! */
+    *priority = 42;
+    treematch->super.type = OMPI_COMM_DIST_GRAPH;
+    return &(treematch->super);
+}
+
+static int mca_topo_treematch_component_register(void)
+{
+    (void)mca_base_component_var_register(&mca_topo_treematch_component.super.topoc_version,
+                                          "reorder_mode", "If set the reordering will be done in a partially distributed way (default=0). If partially-distributed only local knowledge will be used, possibly leading to less accurate reordering.", MCA_BASE_VAR_TYPE_INT,
+                                          NULL, 0, 0, OPAL_INFO_LVL_2,
+                                          MCA_BASE_VAR_SCOPE_READONLY, &mca_topo_treematch_component.reorder_mode);
+    return OMPI_SUCCESS;
+}
+