Alternative to user-defined counters

include/benchmark/benchmark_api.h

@@ -154,6 +154,8 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond);
 #include <stdint.h>
 
 #include <vector>
+#include <map>
+#include <string>
 
 #include "macros.h"
 
@@ -266,6 +268,43 @@ enum BigO {
 // computational complexity for the benchmark.
 typedef double(BigOFunc)(int);
 
+enum CounterType  {
+    CT_Default           = 0,
+    /** Mark the counter as a rate. It will be presented divided by the duration of the benchmark. */
+    CT_Rate              = 1,
+    /** Mark the counter as a thread-average quantity. It will be presented divided by the number of threads. */
+    CT_ThreadAverage     = CT_Rate << 1,
+    CT_ThreadAverageRate = CT_Rate | CT_ThreadAverage
+};
+
+class Counter {
+public:
+   // Allow direct access to both the type and the value.
+  double   value;
+  CounterType type;
+public:
+
+  BENCHMARK_ALWAYS_INLINE
+  Counter(double v = 0, CounterType t = CT_Default)
+      : value(v), type(t) {}
+
+  BENCHMARK_ALWAYS_INLINE Counter& operator=(double v) {
+      value = v;
+      return *this;
+  }
+
+  // Allow Counter to implicitly convert to double to allow use of
+  // binary and compound operators. ie c += c2.
+  BENCHMARK_ALWAYS_INLINE operator       double&() { return value; }
+  BENCHMARK_ALWAYS_INLINE operator const double&() const { return value; }
+
+  // Return 'value' after adjusting for'type'.
+  double FormatValue(double cpu_time, double num_threads) const;
+  std::string FormatType(TimeUnit cpu_time_unit) const;
+};
+
+typedef std::map<std::string, Counter> BenchmarkCounters;
+
 // State is passed to a running Benchmark and contains state for the
 // benchmark to use.
 class State {
@@ -433,6 +472,8 @@ class State {
   BENCHMARK_ALWAYS_INLINE
   size_t iterations() const { return total_iterations_; }
 
+  BenchmarkCounters counters;
+
 private:
   bool started_;
   bool finished_;

include/benchmark/reporter.h

@@ -19,6 +19,7 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <map>
 
 #include "benchmark_api.h"  // For forward declaration of BenchmarkReporter
 
@@ -44,6 +45,7 @@ class BenchmarkReporter {
     Run() :
       error_occurred(false),
       iterations(1),
+      threads(1),
       time_unit(kNanosecond),
       real_accumulated_time(0),
       cpu_accumulated_time(0),
@@ -61,6 +63,7 @@ class BenchmarkReporter {
     std::string error_message;
 
     int64_t iterations;
+    int64_t threads;
     TimeUnit time_unit;
     double real_accumulated_time;
     double cpu_accumulated_time;
@@ -92,6 +95,8 @@ class BenchmarkReporter {
     // Inform print function whether the current run is a complexity report
     bool report_big_o;
     bool report_rms;
+
+    BenchmarkCounters counters;
   };
 
   // Construct a BenchmarkReporter with the output stream set to 'std::cout'

src/CMakeLists.txt

@@ -5,7 +5,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src)
 set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc"
                  "console_reporter.cc" "csv_reporter.cc" "json_reporter.cc"
                  "log.cc" "reporter.cc" "sleep.cc" "string_util.cc"
-                 "sysinfo.cc" "walltime.cc" "complexity.cc")
+                 "sysinfo.cc" "walltime.cc" "complexity.cc" "counter.cc")
+
 # Determine the correct regular expression engine to use
 if(HAVE_STD_REGEX)
   set(RE_FILES "re_std.cc")

src/benchmark.cc

@@ -143,6 +143,7 @@ struct ThreadStats {
     int64_t bytes_processed;
     int64_t items_processed;
     int  complexity_n;
+    BenchmarkCounters counters;
 };
 
 // Timer management class
@@ -794,6 +795,17 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
     total->bytes_processed += st.bytes_processed();
     total->items_processed += st.items_processed();
     total->complexity_n += st.complexity_length_n();
+
+    for (auto const& KV : st.counters) {
+      using Iter = benchmark::BenchmarkCounters::iterator;
+      std::pair<Iter, bool> res = total->counters.insert(KV);
+      if (!res.second) {
+        benchmark::Counter& dest = res.first->second;
+        CHECK_EQ(dest.type, KV.second.type)
+            << "Cannot sum counters with different types";
+        dest += KV.second;
+      }
+    }
   }
 
   timer_manager->Finalize();
@@ -893,6 +905,7 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
         report.report_label = label;
         // Report the total iterations across all threads.
         report.iterations = static_cast<int64_t>(iters) * b.threads;
+        report.threads = b.threads;
         report.time_unit = b.time_unit;
 
         if (!report.error_occurred) {
@@ -916,6 +929,7 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
           report.complexity_n = total.complexity_n;
           report.complexity = b.complexity;
           report.complexity_lambda = b.complexity_lambda;
+          report.counters = std::move(total.counters);
           if(report.complexity != oNone)
             complexity_reports->push_back(report);
         }

src/complexity.cc

@@ -22,6 +22,7 @@
 #include "check.h"
 #include "complexity.h"
 #include "stat.h"
+#include <map>
 
 namespace benchmark {
 
@@ -172,6 +173,16 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
   // All repetitions should be run with the same number of iterations so we
   // can take this information from the first benchmark.
   int64_t const run_iterations = reports.front().iterations;
+  // create stats for user counters
+  std::map< std::string, Stat1_d > counter_stats;
+  for(Run const& r : reports) {
+    for(auto const& KV : r.counters) {
+      auto it = counter_stats.find(KV.first);
+      if(it == counter_stats.end()) {
+        counter_stats.insert({KV.first, Stat1_d{}});
+      }
+    }
+  }
 
   // Populate the accumulators.
   for (Run const& run : reports) {
@@ -184,6 +195,12 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
         Stat1_d(run.cpu_accumulated_time / run.iterations, run.iterations);
     items_per_second_stat += Stat1_d(run.items_per_second, run.iterations);
     bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations);
+    // user counters
+    for(auto const& KV : run.counters) {
+      auto it = counter_stats.find(KV.first);
+      CHECK_NE(it, counter_stats.end());
+      it->second += Stat1_d(KV.second, run.iterations);
+    }
   }
 
   // Get the data from the accumulator to BenchmarkReporter::Run's.
@@ -196,6 +213,10 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
       cpu_accumulated_time_stat.Mean() * run_iterations;
   mean_data.bytes_per_second = bytes_per_second_stat.Mean();
   mean_data.items_per_second = items_per_second_stat.Mean();
+  // user counters
+  for(auto const& kv : counter_stats) {
+    mean_data.counters[kv.first.c_str()] = kv.second.Mean(); /**@todo add the rest of the settings (fmt,flags)*/
+  }
 
   // Only add label to mean/stddev if it is same for all runs
   mean_data.report_label = reports[0].report_label;
@@ -214,6 +235,10 @@ std::vector<BenchmarkReporter::Run> ComputeStats(
   stddev_data.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev();
   stddev_data.bytes_per_second = bytes_per_second_stat.StdDev();
   stddev_data.items_per_second = items_per_second_stat.StdDev();
+  // user counters
+  for(auto const& kv : counter_stats) {
+    stddev_data.counters[kv.first.c_str()] = kv.second.StdDev(); /**@todo add the rest of the settings (fmt,flags)*/
+  }
 
   results.push_back(mean_data);
   results.push_back(stddev_data);

src/console_reporter.cc

@@ -126,6 +126,12 @@ void ConsoleReporter::PrintRunData(const Run& result) {
   if (!result.report_label.empty()) {
     printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str());
   }
+  for (const auto& KV : result.counters) {
+    const Counter& C = KV.second;
+    printer(Out, COLOR_DEFAULT, " %s=%0.0f%s", KV.first.c_str(),
+            C.FormatValue(result.cpu_accumulated_time, result.threads),
+            C.FormatType(result.time_unit).c_str());
+  }
 
   printer(Out, COLOR_DEFAULT, "\n");
 }

src/counter.cc

@@ -0,0 +1,47 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "benchmark/benchmark.h"
+#include "check.h"
+#include "internal_macros.h"
+
+#include <cstring>
+#include <cassert>
+
+namespace benchmark {
+
+double Counter::FormatValue(double cpu_time, double num_threads) const {
+  switch (type) {
+  case CT_Default: return value;
+  case CT_Rate: return value / cpu_time;
+  case CT_ThreadAverage: return value / num_threads;
+  case CT_ThreadAverageRate: return (value / cpu_time) / num_threads;
+  default:
+    BENCHMARK_UNREACHABLE();
+  }
+}
+
+std::string Counter::FormatType(TimeUnit cpu_time_unit) const {
+  std::string const unit_str = GetTimeUnitString(cpu_time_unit);
+  switch (type) {
+  case CT_Default: return "";
+  case CT_Rate: return "/" + unit_str;
+  case CT_ThreadAverage: return "/thread";
+  case CT_ThreadAverageRate: return "/" + unit_str + "/thread";
+  default:
+    BENCHMARK_UNREACHABLE();
+  }
+}
+
+}  // end namespace benchmark

src/internal_macros.h

@@ -2,6 +2,7 @@
 #define BENCHMARK_INTERNAL_MACROS_H_
 
 #include "benchmark/macros.h"
+#include <cassert>
 
 #ifndef __has_feature
 # define __has_feature(x) 0
@@ -15,6 +16,12 @@
 # define BENCHMARK_NORETURN
 #endif
 
+#if defined(__GNUC__)
+# define BENCHMARK_UNREACHABLE() __builtin_unreachable()
+#else
+# define BENCHMARK_UNREACHABLE() assert(false && "unreachable")
+#endif
+
 #if defined(__CYGWIN__)
 # define BENCHMARK_OS_CYGWIN 1
 #elif defined(_WIN32)

src/json_reporter.cc

@@ -29,6 +29,10 @@ namespace benchmark {
 
 namespace {
 
+struct StartDictTag {};
+struct StartListTag {};
+StartListTag StartList{};
+
 std::string FormatKV(std::string const& key, std::string const& value) {
   return StringPrintF("\"%s\": \"%s\"", key.c_str(), value.c_str());
 }
@@ -47,6 +51,19 @@ std::string FormatKV(std::string const& key, int64_t value) {
   return ss.str();
 }
 
+std::string FormatKV(std::string const& key, double value) {
+  std::stringstream ss;
+  ss << '"' << key << "\": " << value;
+  return ss.str();
+}
+
+std::string FormatKV(std::string const& key, StartListTag) {
+  std::stringstream ss;
+  ss << '"' << key << "\": " << "[";
+  return ss.str();
+}
+
+
 int64_t RoundDouble(double v) {
     return static_cast<int64_t>(v + 0.5);
 }
@@ -167,6 +184,24 @@ void JSONReporter::PrintRunData(Run const& run) {
         << indent
         << FormatKV("items_per_second", RoundDouble(run.items_per_second));
   }
+  if (run.counters.size() != 0) {
+      out << ",\n" << indent
+          << FormatKV("counters", StartList);
+      std::string outer_indent = indent + "  ";
+      std::string inner_indent = outer_indent + "  ";
+      const auto first = run.counters.begin();
+      for(auto It=run.counters.begin(); It != run.counters.end(); ++It) {
+        out << (It == first ? "\n" : ",\n");
+        const Counter& C = It->second;
+        const double adjusted_val = C.FormatValue(run.cpu_accumulated_time, run.threads);
+        out << outer_indent << "{\n"
+            << inner_indent << FormatKV("name", It->first) << ",\n"
+            << inner_indent << FormatKV("value", adjusted_val) << ",\n"
+            << inner_indent << FormatKV("type", "unit" + C.FormatType(run.time_unit)) << "\n"
+            << outer_indent << "}";
+      }
+      out << "\n" << indent << "]";
+  }
   if (!run.report_label.empty()) {
     out << ",\n"
         << indent
@@ -175,4 +210,4 @@ void JSONReporter::PrintRunData(Run const& run) {
   out << '\n';
 }
 
-}  // end namespace benchmark
+} // end namespace benchmark

test/benchmark_test.cc

@@ -203,7 +203,7 @@ static void BM_ManualTiming(benchmark::State& state) {
 BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime();
 BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime();
 
-#if __cplusplus >= 201103L
+#ifdef BENCHMARK_HAS_CXX11
 
 template <class ...Args>
 void BM_with_args(benchmark::State& state, Args&&...) {
@@ -218,7 +218,25 @@ void BM_non_template_args(benchmark::State& state, int, double) {
 }
 BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0);
 
-#endif // __cplusplus >= 201103L
+#endif // BENCHMARK_HAS_CXX11
+
+
+static void BM_UserCounter(benchmark::State& state) {
+  static const int depth = 1024;
+  while (state.KeepRunning()) {
+    benchmark::DoNotOptimize(CalculatePi(depth));
+  }
+  state.counters["Foo"] = 1;
+  state.counters["Bar"] = 2;
+  state.counters["Baz"] = 3;
+  state.counters["Bat"] = 5;
+#ifdef BENCHMARK_HAS_CXX11
+  state.counters.insert({{"Foo", 2}, {"Bar", 3}, {"Baz", 5}, {"Bat", 6}});
+#endif
+}
+BENCHMARK(BM_UserCounter)->Threads(8);
+BENCHMARK(BM_UserCounter)->ThreadRange(1, 32);
+BENCHMARK(BM_UserCounter)->ThreadPerCpu();
 
 BENCHMARK_MAIN()