merged Dan Ellsworth's files, updated documentation, removed restore …

…functions, fixed missing items in msr_core.h, finalize_msr no longer takes a useless argument, added comments in header files, removed static from storage functions so raw data can be accessed outside of library, stopped useless batch error messages

CMakeLists.txt

@@ -13,13 +13,13 @@ endif()
 #
 # Add install RPATH to all executables & libraries.
 #
-set(CMAKE_SKIP_BUILD_RPATH FALSE)
-set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
-list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" index)
-if (index LESS 0)
-  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
-endif()
+#set(CMAKE_SKIP_BUILD_RPATH FALSE)
+#set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
+#set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+#list(FIND CMAKE_PLATFORM_IMPLICIT_LINK_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/lib" index)
+#if (index LESS 0)
+#  set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+#endif()
 
 # libmsr headers are in top level include directory
 include_directories(${PROJECT_SOURCE_DIR}/include)
@@ -38,3 +38,4 @@ install(EXPORT libmsr-libs DESTINATION
 add_subdirectory(src)
 add_subdirectory(include)
 add_subdirectory(cmd)
+add_subdirectory(wrap)

README.md

@@ -36,7 +36,7 @@ Please feel free to contact the authors with questions and suggestions.
 
 Authors
 ---------------------
+  * Scott Walker (walker91@llnl.gov)
   * Kathleen Shoga (shoga1@llnl.gov)
   * Barry Rountreee (rountree@llnl.gov)
-  * Scott Walker (walker91@llnl.gov)
   * Lauren Morita (morita4@llnl.gov)

cmd/msr-turbo.c

@@ -47,7 +47,7 @@ int main(int argc, char **argv) {
   } else {
     usage();
   }
-  finalize_msr(1);
+  finalize_msr();
 
   exit(0);
 }

demoapps/powmon/Makefile.am

@@ -0,0 +1,5 @@
+##########################################################################################
+AUTOMAKE_OPTIONS = foreign subdir-objects
+ACLOCAL_FLAGS = -I m4
+
+SUBDIRS = src

demoapps/powmon/README.txt

@@ -0,0 +1,27 @@
+This directory contains 3 libmsr based power monitors.
+
+* powmon - Samples and prints power consumption and allocation per socket for
+systems with 2 sockets
+
+* power_wrapper_static - Samples and prints power consumption and allocation per
+socket for systems with 2 sockets after setting a power cap
+
+* power_wrapper_dynamic - Samples and prints power consumption and allocation
+per socket for systems with 2 sockets and adjusts the cap stepwise every 500
+ms.
+
+----- Building -----
+These tools are built using GNU autotools. You will need to use autotools to
+generate the configure script and the configure script to generate the
+makefiles for your environment.
+
+To generate the configure script: 
+autoconf -if
+
+To generate the makefiles:
+./configure --prefix=<install path>
+
+To make and install the binaries:
+make install
+
+Each monitor includes a usage string. All three monitors are wrappers around some other process that will be executing on the node and include logic so that only one monitor is run per node.

demoapps/powmon/configure.ac

@@ -0,0 +1,20 @@
+m4_define([powmon_version],[1.1.b])
+
+##########################################################################################
+# Auto configure initialization
+AC_INIT(powmon, [powmon_version], [dellswor@cs.uoregon.edu])
+AC_CONFIG_MACRO_DIR([m4])
+AM_INIT_AUTOMAKE([foreign])
+AC_CONFIG_HEADERS([config.h])
+
+##########################################################################################
+# Identify the libmsr to build against
+DE_LIBMSR_VERSION("")
+
+# Check to for extra flags for clock support
+AC_SEARCH_LIBS([clock_gettime],[rt])
+
+##########################################################################################
+# Generate files
+AC_CONFIG_FILES([Makefile src/Makefile])
+AC_OUTPUT

demoapps/powmon/m4/de_libmsr_version.m4

@@ -0,0 +1,21 @@
+#
+# A macro to detect the libmsr version
+#
+# Sets have_msr2 and have_msr1
+
+AU_ALIAS([DE_LIBMSR_VERSION])
+AC_DEFUN([DE_LIBMSR_VERSION],[
+have_msr2="no"
+have_msr1="no"
+AC_CHECK_HEADERS([msr/memhdlr.h],[have_msr2="yes"],[have_msr2="no"])
+if test x$have_msr2 == x"no"
+then
+    AC_CHECK_HEADERS([msr/msr_rapl.h],[have_msr1="yes"
+AC_DEFINE([HAVE_LIBMSR1],1,[libmsr1 is available])
+],[have_msr1="no"])
+else
+    AC_DEFINE([HAVE_LIBMSR2],1,[libmsr2 is available])
+fi
+AM_CONDITIONAL([HAVE_LIBMSR1],[test x$have_msr1 = xyes])
+AM_CONDITIONAL([HAVE_LIBMSR2],[test x$have_msr2 = xyes])
+])

demoapps/powmon/src/Makefile.am

@@ -0,0 +1,17 @@
+##########################################################################################
+AUTOMAKE_OPTIONS = foreign subdir-objects
+ACLOCAL_FLAGS = -I m4
+
+LDFLAGS += -lmsr
+if HAVE_LIBMSR1
+RAPL_SRC = rapl1.c
+endif
+if HAVE_LIBMSR2
+RAPL_SRC = rapl2.c
+endif
+
+bin_PROGRAMS = powmon power_wrapper_static power_wrapper_dynamic
+
+powmon_SOURCES = highlander.c powmon.c $(RAPL_SRC)
+power_wrapper_static_SOURCES = highlander.c power_wrapper_static.c $(RAPL_SRC)
+power_wrapper_dynamic_SOURCES = highlander.c power_wrapper_dynamic.c $(RAPL_SRC)

demoapps/powmon/src/common.c

@@ -0,0 +1,145 @@
+/* common.c
+ *
+ * Copyright (c) 2011-2015, Lawrence Livermore National Security, LLC. LLNL-CODE-645430
+ * Produced at Lawrence Livermore National Laboratory  
+ * Written by  Barry Rountree,   rountree@llnl.gov
+ *             Daniel Ellsworth, ellsworth8@llnl.gov
+ *             Scott Walker,     walker91@llnl.gov
+ *             Kathleen Shoga,   shoga1@llnl.gov
+ *
+ * All rights reserved. 
+ * 
+ * This file is part of libmsr.
+ * 
+ * libmsr is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ * 
+ * libmsr is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+ * details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public License along
+ * with libmsr.  If not, see <http://www.gnu.org/licenses/>. 
+ *
+ * This material is based upon work supported by the U.S. Department
+ * of Energy's Lawrence Livermore National Laboratory. Office of
+ * Science, under Award number DE-AC52-07NA27344.
+ *
+ */
+#include <stdint.h>
+
+#ifdef HAVE_LIBMSR1
+#include <msr/msr_core.h>
+#include <msr/msr_counters.h>
+static uint64_t last_inst[NUM_THREADS];
+static uint64_t last_core[NUM_THREADS];
+static uint64_t last_ref[NUM_THREADS];
+static struct ctr_data c0, c1, c2;
+
+int init_data() {
+    int i;
+    uint64_t inst[NUM_THREADS], core[NUM_THREADS], ref[NUM_THREADS];
+    for(i=0; i<NUM_THREADS; i++) {
+        c0.enable[i] = c1.enable[i] = c2.enable[i] = 1;
+        c0.ring_level[i] = c1.ring_level[i] = c2.ring_level[i] = 3; // usr+os
+        c0.anyThread[i] = c1.anyThread[i] = c2.anyThread[i] = 1;
+        c0.pmi[i] = c1.pmi[i] = c2.pmi[i] = 0;
+
+        last_inst[i]=0;
+        last_core[i]=0;
+        last_ref[i]=0;
+    }
+    // set up the MSRs so that data is collected
+    set_fixed_ctr_ctrl( &c0, &c1, &c2 );
+    // perform a read to init for deltas
+    read_data(inst, core, ref);
+    return 0;
+}
+
+int read_data(uint64_t* inst, uint64_t* core, uint64_t* ref) {
+    int i;
+    get_fixed_ctr_values(&c0, &c1, &c2);
+    for(i=0;i<NUM_THREADS;i++) {
+        //if(inst != NULL) {
+            if(last_inst[i]>c0.value[i]) {
+                inst[i] = c0.value[i] + 0xFFFFFFFFFFFF - last_inst[i];
+            } else {
+                inst[i] = c0.value[i] - last_inst[i];
+            }
+            last_inst[i] = c0.value[i];
+        //}
+        //if(core != NULL) {
+            if(last_core[i]>c1.value[i]) {
+                core[i] = c1.value[i] + 0xFFFFFFFFFFFF - last_core[i];
+            } else {
+                core[i] = c1.value[i]-last_core[i];
+            }
+            last_core[i] = c1.value[i];
+        //}
+        //if(ref != NULL) {
+            ref[i] = c2.value[i]-last_ref[i];
+            last_ref[i] = c2.value[i];
+        //}
+    }
+    return 0;
+}
+#else
+int init_data() {
+    return 0;
+}
+#endif
+
+void take_measurement() {
+#ifdef HAVE_LIBMSR1
+    uint64_t inst[NUM_THREADS], core[NUM_THREADS], ref[NUM_THREADS];
+#endif
+    uint64_t instr0=0, instr1=0;
+    uint64_t core0 =0, core1 =0;
+    int i;
+    double rapl_data[8];
+    pthread_mutex_lock(&mlock);
+
+    // RAPL reads
+    read_rapl_energy_and_power(rapl_data);
+
+#ifdef HAVE_LIBMSR1
+    // counter reads
+    read_data(inst,core,ref);
+    for(i=0;i<8;i++) {
+        instr0 += inst[i];
+        core0  += core[i];
+    }
+    for(i=8;i<16;i++) {
+        instr1 += inst[i];
+        core1  += core[i];
+    }
+#endif
+
+    total_joules += rapl_data[0] + rapl_data[1];
+    limit_joules += rapl_data[2] + rapl_data[3];
+    if(max_watts < rapl_data[4]) { max_watts = rapl_data[4]; }
+    if(max_watts < rapl_data[5]) { max_watts = rapl_data[5]; }
+    if(min_watts > rapl_data[4]) { min_watts = rapl_data[4]; }
+    if(min_watts > rapl_data[5]) { min_watts = rapl_data[5]; }
+    fprintf(logfile, "time:%ld 0joules:%lf 1joules:%lf 0limwatts:%lf 1limwatts:%lf instr0:%lu instr1:%lu core0:%lu core1:%lu\n",now_ms(),rapl_data[0],rapl_data[1],rapl_data[6],rapl_data[7],instr0,instr1,core0,core1);
+    pthread_mutex_unlock(&mlock);
+}
+
+void* power_measurement(void* arg) {
+    struct mstimer timer;
+    // according to the Intel docs, the counter wraps a most once per second
+    // 100 ms should be short enough to always get good information
+    init_msTimer(&timer, 100);
+    init_data();
+    read_rapl_init();
+    start = now_ms();
+
+    timer_sleep(&timer);
+    while(running) {
+        take_measurement();
+        timer_sleep(&timer);
+    }
+}

demoapps/powmon/src/ghighres.c

@@ -0,0 +1,105 @@
+/* ghighres.c
+ *
+ * Copyright (c) 2011-2015, Lawrence Livermore National Security, LLC. LLNL-CODE-645430
+ * Produced at Lawrence Livermore National Laboratory  
+ * Written by  Barry Rountree,   rountree@llnl.gov
+ *             Daniel Ellsworth, ellsworth8@llnl.gov
+ *             Scott Walker,     walker91@llnl.gov
+ *             Kathleen Shoga,   shoga1@llnl.gov
+ *
+ * All rights reserved. 
+ * 
+ * This file is part of libmsr.
+ * 
+ * libmsr is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Lesser General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ * 
+ * libmsr is distributed in the hope that it will be useful, but WITHOUT ANY
+ * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+ * PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+ * details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public License along
+ * with libmsr.  If not, see <http://www.gnu.org/licenses/>. 
+ *
+ * This material is based upon work supported by the U.S. Department
+ * of Energy's Lawrence Livermore National Laboratory. Office of
+ * Science, under Award number DE-AC52-07NA27344.
+ *
+ */
+/*! \file ghighres.c
+ * \brief high resolution sleep function
+ */
+
+#include <time.h>
+#include <sys/select.h>
+
+struct mstimer {
+    unsigned long startms; // when we started tracking the timer
+    uint step;     // which time is the next interval
+    uint interval; // how many ms between firings
+    unsigned long nextms;  // when does the timer expire next
+};
+
+// Get a number of millis from realtime clock
+/*
+unsigned long now_rt_ms() {
+    struct timespec t;
+    clock_gettime(CLOCK_REALTIME, &t);
+    unsigned long sec = t.tv_sec*1000;
+    unsigned long msec = (t.tv_nsec+500000)/1000000;
+    return sec+msec;
+}
+*/
+// Get a number of millis from a monotonic clock
+unsigned long now_ms() {
+    struct timespec t;
+    clock_gettime(CLOCK_REALTIME, &t);
+    unsigned long sec = t.tv_sec*1000;
+    unsigned long msec = (t.tv_nsec+500000)/1000000;
+    return sec+msec;
+}
+
+// Use a select to sleep a given number of millis
+void sleep_ms(long ms) {
+    struct timeval i;
+    i.tv_sec = ms/1000;
+    i.tv_usec = (ms%1000)*1000;
+    select(0, NULL, NULL, NULL, &i);
+}
+
+// Initialize a msTimer
+void init_msTimer(struct mstimer* t, int ms_interval) {
+    t->step=1;
+    t->interval = ms_interval;
+    t->startms = now_ms();
+    t->nextms  = t->startms + t->step*t->interval;
+}
+
+// Initialize a msTimer
+void init_sync_msTimer(struct mstimer* t, int ms_interval, unsigned long start) {
+    t->step=1;
+    t->interval = ms_interval;
+    t->startms = start;
+    t->nextms  = t->startms + t->step*t->interval;
+}
+
+// sleep until timer time has elapsed
+int timer_sleep(struct mstimer* t) {
+    unsigned long now = now_ms();
+    if(now >= t->nextms) {
+        int cadd = 0;
+        while(t->nextms <= now) {
+            cadd++;
+            t->step++;
+            t->nextms = t->startms + t->step * t->interval;
+        }
+        return cadd; // we slipped this many intervals
+    }
+    sleep_ms(t->nextms - now);
+    t->step++;
+    t->nextms = t->startms + t->step * t->interval;
+    return 0;
+}