diff --git a/configure.ac b/configure.ac index f4bbf8ca80..68a8398046 100644 --- a/configure.ac +++ b/configure.ac @@ -841,7 +841,10 @@ OPAL_SEARCH_LIBS_CORE([dirname], [gen]) # Darwin doesn't need -lm, as it's a symlink to libSystem.dylib OPAL_SEARCH_LIBS_CORE([ceil], [m]) -AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair strncpy_s usleep mkfifo dbopen dbm_open statfs statvfs setpgid setenv]) +# -lrt might be needed for clock_gettime +OPAL_SEARCH_LIBS_CORE([clock_gettime], [rt]) + +AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf openpty isatty getpwuid fork waitpid execve pipe ptsname setsid mmap tcgetpgrp posix_memalign strsignal sysconf syslog vsyslog regcmp regexec regfree _NSGetEnviron socketpair strncpy_s usleep mkfifo dbopen dbm_open statfs statvfs setpgid setenv]) # Sanity check: ensure that we got at least one of statfs or statvfs. if test $ac_cv_func_statfs = no -a $ac_cv_func_statvfs = no; then diff --git a/opal/include/opal/sys/amd64/timer.h b/opal/include/opal/sys/amd64/timer.h index 64107efe6d..b382d3a68e 100644 --- a/opal/include/opal/sys/amd64/timer.h +++ b/opal/include/opal/sys/amd64/timer.h @@ -22,6 +22,9 @@ typedef uint64_t opal_timer_t; +/* Using RDTSC(P) results in non-monotonic timers across cores */ +#undef OPAL_TIMER_MONOTONIC +#define OPAL_TIMER_MONOTONIC 0 #if OMPI_GCC_INLINE_ASSEMBLY diff --git a/opal/include/opal/sys/timer.h b/opal/include/opal/sys/timer.h index cc8b92207a..217b3e01d0 100644 --- a/opal/include/opal/sys/timer.h +++ b/opal/include/opal/sys/timer.h @@ -70,6 +70,9 @@ * *********************************************************************/ +/* By default we suppose all timers are monotonic per node. */ +#define OPAL_TIMER_MONOTONIC 1 + BEGIN_C_DECLS /* If you update this list, you probably also want to update diff --git a/opal/mca/timer/base/base.h b/opal/mca/timer/base/base.h index f058a45589..b45bce0413 100644 --- a/opal/mca/timer/base/base.h +++ b/opal/mca/timer/base/base.h @@ -35,7 +35,12 @@ BEGIN_C_DECLS * Framework structure declaration */ OPAL_DECLSPEC extern mca_base_framework_t opal_timer_base_framework; - + +/** + * MCA param to force monotonic timers. + */ +OPAL_DECLSPEC extern int mca_timer_base_monotonic; + END_C_DECLS /* include implementation to call */ diff --git a/opal/mca/timer/base/timer_base_open.c b/opal/mca/timer/base/timer_base_open.c index 71c8b9df72..ef1676a768 100644 --- a/opal/mca/timer/base/timer_base_open.c +++ b/opal/mca/timer/base/timer_base_open.c @@ -22,6 +22,7 @@ #include "opal/constants.h" #include "opal/mca/timer/base/base.h" +int mca_timer_base_monotonic = 1; /* * The following file was created by configure. It contains extern @@ -30,9 +31,23 @@ */ #include "opal/mca/timer/base/static-components.h" +static int mca_timer_base_register(mca_base_register_flag_t flags) +{ + /* figure out which bcol and sbgp components will actually be used */ + /* get list of sub-grouping functions to use */ + (void) mca_base_var_register("opal", "timer", "require", "monotonic", + "Node-level monotonic timer required (default yes)", + MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_LOCAL, + &mca_timer_base_monotonic); + + return OPAL_SUCCESS; +} + /* * Globals */ /* Use default register/open/close functions */ -MCA_BASE_FRAMEWORK_DECLARE(opal, timer, NULL, NULL, NULL, NULL, +MCA_BASE_FRAMEWORK_DECLARE(opal, timer, "OPAL OS timer", mca_timer_base_register, NULL, NULL, mca_timer_base_static_components, 0); diff --git a/opal/mca/timer/linux/Makefile.am b/opal/mca/timer/linux/Makefile.am index 7568a98864..e2320ec779 100644 --- a/opal/mca/timer/linux/Makefile.am +++ b/opal/mca/timer/linux/Makefile.am @@ -2,21 +2,23 @@ # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana # University Research and Technology # Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University +# Copyright (c) 2004-2014 The University of Tennessee and The University # of Tennessee Research Foundation. All rights # reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, # University of Stuttgart. All rights reserved. # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. # $COPYRIGHT$ -# +# # Additional copyrights may follow -# +# # $HEADER$ # +dist_ompidata_DATA = help-opal-timer-linux.txt + noinst_LTLIBRARIES = libmca_timer_linux.la libmca_timer_linux_la_SOURCES = \ diff --git a/opal/mca/timer/linux/help-opal-timer-linux.txt b/opal/mca/timer/linux/help-opal-timer-linux.txt new file mode 100644 index 0000000000..c4cb122a93 --- /dev/null +++ b/opal/mca/timer/linux/help-opal-timer-linux.txt @@ -0,0 +1,17 @@ +# -*- text -*- +# +# Copyright (c) 2014 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is the US/English help file for Open MPI's Linux timer support. +# +[monotonic not supported] +A monotonic timer has been requested by the user, but could not be found on the +system. Falling back to a non-monotonic timer (such as RDTSC). + diff --git a/opal/mca/timer/linux/timer_linux.h b/opal/mca/timer/linux/timer_linux.h index 6bdffcb0f7..6bf05c9f5d 100644 --- a/opal/mca/timer/linux/timer_linux.h +++ b/opal/mca/timer/linux/timer_linux.h @@ -2,17 +2,17 @@ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ @@ -24,39 +24,14 @@ OPAL_DECLSPEC extern opal_timer_t opal_timer_linux_freq; -static inline opal_timer_t -opal_timer_base_get_cycles(void) -{ -#if OPAL_HAVE_SYS_TIMER_GET_CYCLES - return opal_sys_timer_get_cycles(); -#else - return 0; -#endif -} - - -static inline opal_timer_t -opal_timer_base_get_usec(void) -{ -#if OPAL_HAVE_SYS_TIMER_GET_CYCLES - /* freq is in Hz, so this gives usec */ - return opal_sys_timer_get_cycles() * 1000000 / opal_timer_linux_freq; -#else - return 0; -#endif -} - - -static inline opal_timer_t -opal_timer_base_get_freq(void) -{ - return opal_timer_linux_freq; -} +OPAL_DECLSPEC extern opal_timer_t (*opal_timer_base_get_cycles)(void); +OPAL_DECLSPEC extern opal_timer_t (*opal_timer_base_get_usec)(void); +OPAL_DECLSPEC extern opal_timer_t opal_timer_base_get_freq(void); #define OPAL_TIMER_CYCLE_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES #define OPAL_TIMER_CYCLE_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES -#define OPAL_TIMER_USEC_NATIVE 0 +#define OPAL_TIMER_USEC_NATIVE OPAL_HAVE_SYS_TIMER_GET_CYCLES #define OPAL_TIMER_USEC_SUPPORTED OPAL_HAVE_SYS_TIMER_GET_CYCLES #endif diff --git a/opal/mca/timer/linux/timer_linux_component.c b/opal/mca/timer/linux/timer_linux_component.c index a5ac4ff2c2..5449597102 100644 --- a/opal/mca/timer/linux/timer_linux_component.c +++ b/opal/mca/timer/linux/timer_linux_component.c @@ -1,11 +1,12 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University + * Copyright (c) 2004-2014 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. @@ -15,23 +16,35 @@ * reserved. * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * $COPYRIGHT$ - * + * * Additional copyrights may follow - * + * * $HEADER$ */ #include "opal_config.h" -#ifdef HAVE_STRING_H #include -#endif #include "opal/mca/timer/timer.h" +#include "opal/mca/timer/base/base.h" #include "opal/mca/timer/linux/timer_linux.h" #include "opal/constants.h" -opal_timer_t opal_timer_linux_freq; +static opal_timer_t opal_timer_base_get_cycles_sys_timer(void); +static opal_timer_t opal_timer_base_get_usec_sys_timer(void); + +#if OPAL_HAVE_CLOCK_GETTIME +static opal_timer_t opal_timer_base_get_cycles_clock_gettime(void); +static opal_timer_t opal_timer_base_get_usec_clock_gettime(void); +opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_base_get_cycles_clock_gettime; +opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_base_get_usec_clock_gettime; +#else +opal_timer_t (*opal_timer_base_get_cycles)(void) = opal_timer_base_get_cycles_sys_timer; +opal_timer_t (*opal_timer_base_get_usec)(void) = opal_timer_base_get_usec_sys_timer; +#endif /* OPAL_HAVE_CLOCK_GETTIME */ + +opal_timer_t opal_timer_linux_freq = {0}; static int opal_timer_linux_open(void); @@ -68,6 +81,9 @@ find_info(FILE* fp, char *str, char *buf, size_t buflen) /* we found the line. Now eat everything up to, including, and one past the : */ for (tmp = buf ; (*tmp != '\0') && (*tmp != ':') ; ++tmp) ; + if (*tmp == '\0') { + continue; + } for ( ++tmp ; *tmp == ' ' ; ++tmp); if ('\0' != *tmp) { return tmp; @@ -78,8 +94,7 @@ find_info(FILE* fp, char *str, char *buf, size_t buflen) return NULL; } -int -opal_timer_linux_open(void) +static int opal_timer_linux_find_freq(void) { FILE *fp; char *loc; @@ -135,3 +150,78 @@ opal_timer_linux_open(void) return OPAL_SUCCESS; } + +int opal_timer_linux_open(void) +{ + int ret = OPAL_SUCCESS; + + if(mca_timer_base_monotonic) { +#if OPAL_HAVE_CLOCK_GETTIME + struct timespec res; + if( 0 == clock_getres(CLOCK_MONOTONIC, &res)) { + opal_timer_linux_freq = 1.e9; + opal_timer_base_get_cycles = opal_timer_base_get_cycles_clock_gettime; + opal_timer_base_get_usec = opal_timer_base_get_usec_clock_gettime; + return ret; + } +#else +#if (0 == OPAL_TIMER_MONOTONIC) + /* Monotonic time requested but cannot be found. Complain! */ + opal_show_help("help-opal-timer-linux.txt", "monotonic not supported", 1); +#endif /* (0 == OPAL_TIMER_MONOTONIC) */ +#endif + } + ret = opal_timer_linux_find_freq(); + opal_timer_base_get_cycles = opal_timer_base_get_cycles_sys_timer; + opal_timer_base_get_usec = opal_timer_base_get_usec_sys_timer; + return ret; +} + +#if OPAL_HAVE_CLOCK_GETTIME +opal_timer_t opal_timer_base_get_usec_clock_gettime(void) +{ + struct timespec tp; + + if( 0 == clock_gettime(CLOCK_MONOTONIC, &tp) ) { + return (tp.tv_sec * 1e6 + tp.tv_nsec/1000); + } + return 0; +} + +opal_timer_t opal_timer_base_get_cycles_clock_gettime(void) +{ + struct timespec tp; + + if( 0 == clock_gettime(CLOCK_MONOTONIC, &tp) ) { + return (tp.tv_sec * 1e9 + tp.tv_nsec); + } + return 0; +} +#endif /* OPAL_HAVE_CLOCK_GETTIME */ + +opal_timer_t opal_timer_base_get_cycles_sys_timer(void) +{ +#if OPAL_HAVE_SYS_TIMER_GET_CYCLES + return opal_sys_timer_get_cycles(); +#else + return 0; +#endif +} + + +opal_timer_t opal_timer_base_get_usec_sys_timer(void) +{ +#if OPAL_HAVE_SYS_TIMER_GET_CYCLES + /* freq is in Hz, so this gives usec */ + return opal_sys_timer_get_cycles() * 1000000 / opal_timer_linux_freq; +#else + return 0; +#endif +} + +opal_timer_t opal_timer_base_get_freq(void) +{ + return opal_timer_linux_freq; +} + +