Update the Linux timer per the master and George's patch on the devel mailing list.

configure.ac

@@ -841,7 +841,10 @@ OPAL_SEARCH_LIBS_CORE([dirname], [gen])
 # Darwin doesn't need -lm, as it's a symlink to libSystem.dylib
 OPAL_SEARCH_LIBS_CORE([ceil], [m])
 
-AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair strncpy_s usleep mkfifo dbopen dbm_open statfs statvfs setpgid  setenv])
+# -lrt might be needed for clock_gettime
+OPAL_SEARCH_LIBS_CORE([clock_gettime], [rt])
+
+AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair strncpy_s usleep mkfifo dbopen dbm_open statfs statvfs setpgid setenv])
 
 # Sanity check: ensure that we got at least one of statfs or statvfs.
 if test $ac_cv_func_statfs = no -a $ac_cv_func_statvfs = no; then

opal/include/opal/sys/amd64/timer.h

@@ -22,6 +22,9 @@
 
 typedef uint64_t opal_timer_t;
 
+/* Using RDTSC(P) results in non-monotonic timers across cores */
+#undef OPAL_TIMER_MONOTONIC
+#define OPAL_TIMER_MONOTONIC 0
 
 #if OMPI_GCC_INLINE_ASSEMBLY
 

opal/include/opal/sys/timer.h

@@ -70,6 +70,9 @@
  *
  *********************************************************************/
 
+/* By default we suppose all timers are monotonic per node. */
+#define OPAL_TIMER_MONOTONIC 1
+
 BEGIN_C_DECLS
 
 /* If you update this list, you probably also want to update

opal/mca/timer/base/base.h

@@ -35,7 +35,12 @@ BEGIN_C_DECLS
  * Framework structure declaration
  */
 OPAL_DECLSPEC extern mca_base_framework_t opal_timer_base_framework;
-
+
+/**
+ * MCA param to force monotonic timers.
+ */
+OPAL_DECLSPEC extern int mca_timer_base_monotonic;
+
 END_C_DECLS
 
 /* include implementation to call */

opal/mca/timer/base/timer_base_open.c

@@ -22,6 +22,7 @@
 #include "opal/constants.h"
 #include "opal/mca/timer/base/base.h"
 
+int mca_timer_base_monotonic = 1;
 
 /*
  * The following file was created by configure.  It contains extern
@@ -30,9 +31,23 @@
  */
 #include "opal/mca/timer/base/static-components.h"
 
+static int mca_timer_base_register(mca_base_register_flag_t flags)
+{
+    /* figure out which bcol and sbgp components will actually be used */
+    /* get list of sub-grouping functions to use */
+    (void) mca_base_var_register("opal", "timer", "require", "monotonic",
+                                 "Node-level monotonic timer required (default yes)",
+                                 MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+                                 OPAL_INFO_LVL_9,
+                                 MCA_BASE_VAR_SCOPE_LOCAL,
+                                 &mca_timer_base_monotonic);
+
+    return OPAL_SUCCESS;
+}
+
 /*
  * Globals
  */
 /* Use default register/open/close functions */
-MCA_BASE_FRAMEWORK_DECLARE(opal, timer, NULL, NULL, NULL, NULL,
+MCA_BASE_FRAMEWORK_DECLARE(opal, timer, "OPAL OS timer", mca_timer_base_register, NULL, NULL,
                            mca_timer_base_static_components, 0);

opal/mca/timer/linux/Makefile.am

@@ -2,21 +2,23 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2014 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
-# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
 # Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
 # $COPYRIGHT$
-# 
+#
 # Additional copyrights may follow
-# 
+#
 # $HEADER$
 #
 
+dist_ompidata_DATA = help-opal-timer-linux.txt
+
 noinst_LTLIBRARIES = libmca_timer_linux.la
 
 libmca_timer_linux_la_SOURCES = \

opal/mca/timer/linux/help-opal-timer-linux.txt

@@ -0,0 +1,17 @@
+# -*- text -*-
+#
+# Copyright (c) 2014      The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+# This is the US/English help file for Open MPI's Linux timer support.
+#
+[monotonic not supported]
+A monotonic timer has been requested by the user, but could not be found on the
+system. Falling back to a non-monotonic timer (such as RDTSC).
+

opal/mca/timer/linux/timer_linux.h

@@ -2,17 +2,17 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * $COPYRIGHT$
- * 
+ *
  * Additional copyrights may follow
- * 
+ *
  * $HEADER$
  */
 
@@ -24,39 +24,14 @@
 
 OPAL_DECLSPEC extern opal_timer_t opal_timer_linux_freq;
 
-static inline opal_timer_t
-opal_timer_base_get_cycles(void)
-{
-#if OPAL_HAVE_SYS_TIMER_GET_CYCLES
-    return opal_sys_timer_get_cycles();
-#else
-    return 0;
-#endif
-}
-
-
-static inline opal_timer_t
-opal_timer_base_get_usec(void)
-{
-#if OPAL_HAVE_SYS_TIMER_GET_CYCLES
-    /* freq is in Hz, so this gives usec */
-    return opal_sys_timer_get_cycles() * 1000000  / opal_timer_linux_freq;
-#else
-    return 0;
-#endif
-}    
-
-
-static inline opal_timer_t
-opal_timer_base_get_freq(void)
-{
-    return opal_timer_linux_freq;
-}
+OPAL_DECLSPEC extern opal_timer_t (*opal_timer_base_get_cycles)(void);
+OPAL_DECLSPEC extern opal_timer_t (*opal_timer_base_get_usec)(void);
 
+OPAL_DECLSPEC extern opal_timer_t opal_timer_base_get_freq(void);
 
 #define OPAL_TIMER_CYCLE_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES
 #define OPAL_TIMER_CYCLE_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES
-#define OPAL_TIMER_USEC_NATIVE 0
+#define OPAL_TIMER_USEC_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES
 #define OPAL_TIMER_USEC_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES
 
 #endif

opal/mca/timer/linux/timer_linux_component.c

@@ -1,11 +1,12 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2014 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
@@ -15,23 +16,35 @@
  *                         reserved.
  * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
- * 
+ *
  * Additional copyrights may follow
- * 
+ *
  * $HEADER$
  */
 
 #include "opal_config.h"
 
-#ifdef HAVE_STRING_H
 #include <string.h>
-#endif
 
 #include "opal/mca/timer/timer.h"
+#include "opal/mca/timer/base/base.h"
 #include "opal/mca/timer/linux/timer_linux.h"
 #include "opal/constants.h"
 
-opal_timer_t opal_timer_linux_freq;
+static opal_timer_t opal_timer_base_get_cycles_sys_timer(void);
+static opal_timer_t opal_timer_base_get_usec_sys_timer(void);
+
+#if OPAL_HAVE_CLOCK_GETTIME
+static opal_timer_t opal_timer_base_get_cycles_clock_gettime(void);
+static opal_timer_t opal_timer_base_get_usec_clock_gettime(void);
+opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_base_get_cycles_clock_gettime;
+opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_base_get_usec_clock_gettime;
+#else
+opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_base_get_cycles_sys_timer;
+opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_base_get_usec_sys_timer;
+#endif  /* OPAL_HAVE_CLOCK_GETTIME */
+
+opal_timer_t opal_timer_linux_freq = {0};
 
 static int opal_timer_linux_open(void);
 
@@ -68,6 +81,9 @@ find_info(FILE* fp, char *str, char *buf, size_t buflen)
             /* we found the line.  Now eat everything up to,
                including, and one past the : */
             for (tmp = buf ; (*tmp != '\0') && (*tmp != ':') ; ++tmp) ;
+            if (*tmp == '\0') {
+                continue;
+            }
             for ( ++tmp ; *tmp == ' ' ; ++tmp);
             if ('\0' != *tmp) {
                 return tmp;
@@ -78,8 +94,7 @@ find_info(FILE* fp, char *str, char *buf, size_t buflen)
     return NULL;
 }
 
-int
-opal_timer_linux_open(void)
+static int opal_timer_linux_find_freq(void)
 {
     FILE *fp;
     char *loc;
@@ -135,3 +150,78 @@ opal_timer_linux_open(void)
 
     return OPAL_SUCCESS;
 }
+
+int opal_timer_linux_open(void)
+{
+    int ret = OPAL_SUCCESS;
+
+    if(mca_timer_base_monotonic) {
+#if OPAL_HAVE_CLOCK_GETTIME
+        struct timespec res;
+        if( 0 == clock_getres(CLOCK_MONOTONIC, &res)) {
+            opal_timer_linux_freq = 1.e9;
+            opal_timer_base_get_cycles = opal_timer_base_get_cycles_clock_gettime;
+            opal_timer_base_get_usec = opal_timer_base_get_usec_clock_gettime;
+            return ret;
+        }
+#else
+#if (0 == OPAL_TIMER_MONOTONIC)
+        /* Monotonic time requested but cannot be found. Complain! */
+        opal_show_help("help-opal-timer-linux.txt", "monotonic not supported", 1);
+#endif  /* (0 == OPAL_TIMER_MONOTONIC) */
+#endif
+    }
+    ret = opal_timer_linux_find_freq();
+    opal_timer_base_get_cycles = opal_timer_base_get_cycles_sys_timer;
+    opal_timer_base_get_usec = opal_timer_base_get_usec_sys_timer;
+    return ret;
+}
+
+#if OPAL_HAVE_CLOCK_GETTIME
+opal_timer_t opal_timer_base_get_usec_clock_gettime(void)
+{
+    struct timespec tp;
+
+    if( 0 == clock_gettime(CLOCK_MONOTONIC, &tp) ) {
+        return (tp.tv_sec * 1e6 + tp.tv_nsec/1000);
+    }
+    return 0;
+}
+
+opal_timer_t opal_timer_base_get_cycles_clock_gettime(void)
+{
+    struct timespec tp;
+
+    if( 0 == clock_gettime(CLOCK_MONOTONIC, &tp) ) {
+        return (tp.tv_sec * 1e9 + tp.tv_nsec);
+    }
+    return 0;
+}
+#endif  /* OPAL_HAVE_CLOCK_GETTIME */
+
+opal_timer_t opal_timer_base_get_cycles_sys_timer(void)
+{
+#if OPAL_HAVE_SYS_TIMER_GET_CYCLES
+    return opal_sys_timer_get_cycles();
+#else
+    return 0;
+#endif
+}
+
+
+opal_timer_t opal_timer_base_get_usec_sys_timer(void)
+{
+#if OPAL_HAVE_SYS_TIMER_GET_CYCLES
+    /* freq is in Hz, so this gives usec */
+    return opal_sys_timer_get_cycles() * 1000000  / opal_timer_linux_freq;
+#else
+    return 0;
+#endif
+}
+
+opal_timer_t opal_timer_base_get_freq(void)
+{
+    return opal_timer_linux_freq;
+}
+
+