From 60302ded47e7f20ced728f06bb4d71708ad2c0c3 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Wed, 17 Aug 2016 20:42:32 -0600 Subject: [PATCH 1/3] implement different custom counters --- include/benchmark/benchmark_api.h | 41 +++++++++++++++++ include/benchmark/reporter.h | 5 ++ src/CMakeLists.txt | 3 +- src/benchmark.cc | 14 ++++++ src/complexity.cc | 25 ++++++++++ src/console_reporter.cc | 6 +++ src/counter.cc | 48 +++++++++++++++++++ src/json_reporter.cc | 37 ++++++++++++++- test/benchmark_test.cc | 22 ++++++++- test/cxx03_test.cc | 9 ++++ test/reporter_output_test.cc | 76 ++++++++++++++++++++++++++++++- 11 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 src/counter.cc diff --git a/include/benchmark/benchmark_api.h b/include/benchmark/benchmark_api.h index ddfcb0142b..f585ca0336 100644 --- a/include/benchmark/benchmark_api.h +++ b/include/benchmark/benchmark_api.h @@ -154,6 +154,8 @@ BENCHMARK(BM_test)->Unit(benchmark::kMillisecond); #include #include +#include +#include #include "macros.h" @@ -266,6 +268,43 @@ enum BigO { // computational complexity for the benchmark. typedef double(BigOFunc)(int); +enum CounterType { + CT_Default = 0, + /** Mark the counter as a rate. It will be presented divided by the duration of the benchmark. */ + CT_Rate = 1, + /** Mark the counter as a thread-average quantity. It will be presented divided by the number of threads. */ + CT_ThreadAverage = CT_Rate << 1, + CT_ThreadAverageRate = CT_Rate | CT_ThreadAverage +}; + +class Counter { +public: + // Allow direct access to both the type and the value. + double value; + CounterType type; +public: + + BENCHMARK_ALWAYS_INLINE + Counter(double v = 0, CounterType t = CT_Default) + : value(v), type(t) {} + + BENCHMARK_ALWAYS_INLINE Counter& operator=(double v) { + value = v; + return *this; + } + + // Allow Counter to implicitly convert to double to allow use of + // binary and compound operators. ie c += c2. + BENCHMARK_ALWAYS_INLINE operator double&() { return value; } + BENCHMARK_ALWAYS_INLINE operator const double&() const { return value; } + + // Return 'value' after adjusting for'type'. + double FormatValue(double cpu_time, double num_threads) const; + std::string FormatType(TimeUnit cpu_time_unit) const; +}; + +typedef std::map BenchmarkCounters; + // State is passed to a running Benchmark and contains state for the // benchmark to use. class State { @@ -433,6 +472,8 @@ class State { BENCHMARK_ALWAYS_INLINE size_t iterations() const { return total_iterations_; } + BenchmarkCounters counters; + private: bool started_; bool finished_; diff --git a/include/benchmark/reporter.h b/include/benchmark/reporter.h index 6d36c41b1c..83527be95e 100644 --- a/include/benchmark/reporter.h +++ b/include/benchmark/reporter.h @@ -19,6 +19,7 @@ #include #include #include +#include #include "benchmark_api.h" // For forward declaration of BenchmarkReporter @@ -44,6 +45,7 @@ class BenchmarkReporter { Run() : error_occurred(false), iterations(1), + threads(1), time_unit(kNanosecond), real_accumulated_time(0), cpu_accumulated_time(0), @@ -61,6 +63,7 @@ class BenchmarkReporter { std::string error_message; int64_t iterations; + int64_t threads; TimeUnit time_unit; double real_accumulated_time; double cpu_accumulated_time; @@ -92,6 +95,8 @@ class BenchmarkReporter { // Inform print function whether the current run is a complexity report bool report_big_o; bool report_rms; + + BenchmarkCounters counters; }; // Construct a BenchmarkReporter with the output stream set to 'std::cout' diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6dab64b73b..e13c6d2b26 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,7 +5,8 @@ include_directories(${PROJECT_SOURCE_DIR}/src) set(SOURCE_FILES "benchmark.cc" "colorprint.cc" "commandlineflags.cc" "console_reporter.cc" "csv_reporter.cc" "json_reporter.cc" "log.cc" "reporter.cc" "sleep.cc" "string_util.cc" - "sysinfo.cc" "walltime.cc" "complexity.cc") + "sysinfo.cc" "walltime.cc" "complexity.cc" "counter.cc") + # Determine the correct regular expression engine to use if(HAVE_STD_REGEX) set(RE_FILES "re_std.cc") diff --git a/src/benchmark.cc b/src/benchmark.cc index 3a199e506e..ea48bedb5c 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -143,6 +143,7 @@ struct ThreadStats { int64_t bytes_processed; int64_t items_processed; int complexity_n; + BenchmarkCounters counters; }; // Timer management class @@ -794,6 +795,17 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, total->bytes_processed += st.bytes_processed(); total->items_processed += st.items_processed(); total->complexity_n += st.complexity_length_n(); + + for (auto const& KV : st.counters) { + using Iter = benchmark::BenchmarkCounters::iterator; + std::pair res = total->counters.insert(KV); + if (!res.second) { + benchmark::Counter& dest = res.first->second; + CHECK_EQ(dest.Type(), KV.second.Type()) + << "Cannot sum counters with different types"; + dest += KV.second; + } + } } timer_manager->Finalize(); @@ -893,6 +905,7 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.report_label = label; // Report the total iterations across all threads. report.iterations = static_cast(iters) * b.threads; + report.threads = b.threads; report.time_unit = b.time_unit; if (!report.error_occurred) { @@ -916,6 +929,7 @@ RunBenchmark(const benchmark::internal::Benchmark::Instance& b, report.complexity_n = total.complexity_n; report.complexity = b.complexity; report.complexity_lambda = b.complexity_lambda; + report.counters = std::move(total.counters); if(report.complexity != oNone) complexity_reports->push_back(report); } diff --git a/src/complexity.cc b/src/complexity.cc index 7d5579430b..e22007a038 100644 --- a/src/complexity.cc +++ b/src/complexity.cc @@ -22,6 +22,7 @@ #include "check.h" #include "complexity.h" #include "stat.h" +#include namespace benchmark { @@ -172,6 +173,16 @@ std::vector ComputeStats( // All repetitions should be run with the same number of iterations so we // can take this information from the first benchmark. int64_t const run_iterations = reports.front().iterations; + // create stats for user counters + std::map< std::string, Stat1_d > counter_stats; + for(Run const& r : reports) { + for(auto const& KV : r.counters) { + auto it = counter_stats.find(KV.first); + if(it == counter_stats.end()) { + counter_stats.insert({KV.first, Stat1_d{}}); + } + } + } // Populate the accumulators. for (Run const& run : reports) { @@ -184,6 +195,12 @@ std::vector ComputeStats( Stat1_d(run.cpu_accumulated_time / run.iterations, run.iterations); items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); + // user counters + for(auto const& KV : run.counters) { + auto it = counter_stats.find(KV.first); + CHECK_NE(it, counter_stats.end()); + it->second += Stat1_d(KV.second, run.iterations); + } } // Get the data from the accumulator to BenchmarkReporter::Run's. @@ -196,6 +213,10 @@ std::vector ComputeStats( cpu_accumulated_time_stat.Mean() * run_iterations; mean_data.bytes_per_second = bytes_per_second_stat.Mean(); mean_data.items_per_second = items_per_second_stat.Mean(); + // user counters + for(auto const& kv : counter_stats) { + mean_data.counters[kv.first.c_str()] = kv.second.Mean(); /**@todo add the rest of the settings (fmt,flags)*/ + } // Only add label to mean/stddev if it is same for all runs mean_data.report_label = reports[0].report_label; @@ -214,6 +235,10 @@ std::vector ComputeStats( stddev_data.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); stddev_data.items_per_second = items_per_second_stat.StdDev(); + // user counters + for(auto const& kv : counter_stats) { + stddev_data.counters[kv.first.c_str()] = kv.second.StdDev(); /**@todo add the rest of the settings (fmt,flags)*/ + } results.push_back(mean_data); results.push_back(stddev_data); diff --git a/src/console_reporter.cc b/src/console_reporter.cc index a663b93675..6e77b55693 100644 --- a/src/console_reporter.cc +++ b/src/console_reporter.cc @@ -126,6 +126,12 @@ void ConsoleReporter::PrintRunData(const Run& result) { if (!result.report_label.empty()) { printer(Out, COLOR_DEFAULT, " %s", result.report_label.c_str()); } + for (const auto& KV : result.counters) { + const Counter& C = KV.second; + printer(Out, COLOR_DEFAULT, " %s=%0.0f%s", KV.first.c_str(), + C.FormatValue(result.cpu_accumulated_time, result.threads), + C.FormatType(result.time_unit).c_str()); + } printer(Out, COLOR_DEFAULT, "\n"); } diff --git a/src/counter.cc b/src/counter.cc new file mode 100644 index 0000000000..ccf258cc1a --- /dev/null +++ b/src/counter.cc @@ -0,0 +1,48 @@ +// Copyright 2015 Google Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "benchmark/benchmark.h" +#include "check.h" +#include "internal_macros.h" + +#include +#include + +namespace benchmark { + +double Counter::FormatValue(double cpu_time, double num_threads) const { + switch (type) { + case CT_Default: return value; + case CT_Rate: return value / cpu_time; + case CT_ThreadAverage: return value / num_threads; + case CT_ThreadAverageRate: return (value / cpu_time) / num_threads; + default: + assert(false && "unreachable"); + } +} + +std::string Counter::FormatType(TimeUnit cpu_time_unit) const { + std::string str; + std::string const unit_str = GetTimeUnitString(cpu_time_unit); + switch (type) { + case CT_Default: return ""; + case CT_Rate: return "/" + unit_str; + case CT_ThreadAverage: return "/thread"; + case CT_ThreadAverageRate: return "/" + unit_str + "/thread"; + default: + assert(false && "unreachable"); + } +} + +} // end namespace benchmark diff --git a/src/json_reporter.cc b/src/json_reporter.cc index 485d30524a..15b05a87af 100644 --- a/src/json_reporter.cc +++ b/src/json_reporter.cc @@ -29,6 +29,10 @@ namespace benchmark { namespace { +struct StartDictTag {}; +struct StartListTag {}; +StartListTag StartList{}; + std::string FormatKV(std::string const& key, std::string const& value) { return StringPrintF("\"%s\": \"%s\"", key.c_str(), value.c_str()); } @@ -47,6 +51,19 @@ std::string FormatKV(std::string const& key, int64_t value) { return ss.str(); } +std::string FormatKV(std::string const& key, double value) { + std::stringstream ss; + ss << '"' << key << "\": " << value; + return ss.str(); +} + +std::string FormatKV(std::string const& key, StartListTag) { + std::stringstream ss; + ss << '"' << key << "\": " << "["; + return ss.str(); +} + + int64_t RoundDouble(double v) { return static_cast(v + 0.5); } @@ -167,6 +184,24 @@ void JSONReporter::PrintRunData(Run const& run) { << indent << FormatKV("items_per_second", RoundDouble(run.items_per_second)); } + if (run.counters.size() != 0) { + out << ",\n" << indent + << FormatKV("counters", StartList); + std::string outer_indent = indent + " "; + std::string inner_indent = outer_indent + " "; + const auto first = run.counters.begin(); + for(auto It=run.counters.begin(); It != run.counters.end(); ++It) { + out << (It == first ? "\n" : ",\n"); + const Counter& C = It->second; + const double adjusted_val = C.FormatValue(run.cpu_accumulated_time, run.threads); + out << outer_indent << "{\n" + << inner_indent << FormatKV("name", It->first) << ",\n" + << inner_indent << FormatKV("value", adjusted_val) << ",\n" + << inner_indent << FormatKV("type", "unit" + C.FormatType(run.time_unit)) << "\n" + << outer_indent << "}"; + } + out << "\n" << indent << "]"; + } if (!run.report_label.empty()) { out << ",\n" << indent @@ -175,4 +210,4 @@ void JSONReporter::PrintRunData(Run const& run) { out << '\n'; } -} // end namespace benchmark +} // end namespace benchmark diff --git a/test/benchmark_test.cc b/test/benchmark_test.cc index fe7d82c6a8..9010263e2f 100644 --- a/test/benchmark_test.cc +++ b/test/benchmark_test.cc @@ -203,7 +203,7 @@ static void BM_ManualTiming(benchmark::State& state) { BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseRealTime(); BENCHMARK(BM_ManualTiming)->Range(1, 1 << 14)->UseManualTime(); -#if __cplusplus >= 201103L +#ifdef BENCHMARK_HAS_CXX11 template void BM_with_args(benchmark::State& state, Args&&...) { @@ -218,7 +218,25 @@ void BM_non_template_args(benchmark::State& state, int, double) { } BENCHMARK_CAPTURE(BM_non_template_args, basic_test, 0, 0); -#endif // __cplusplus >= 201103L +#endif // BENCHMARK_HAS_CXX11 + + +static void BM_UserCounter(benchmark::State& state) { + static const int depth = 1024; + while (state.KeepRunning()) { + benchmark::DoNotOptimize(CalculatePi(depth)); + } + state.counters["Foo"] = 1; + state.counters["Bar"] = 2; + state.counters["Baz"] = 3; + state.counters["Bat"] = 5; +#ifdef BENCHMARK_HAS_CXX11 + state.counters.insert({{"Foo", 2}, {"Bar", 3}, {"Baz", 5}, {"Bat", 6}}); +#endif +} +BENCHMARK(BM_UserCounter)->Threads(8); +BENCHMARK(BM_UserCounter)->ThreadRange(1, 32); +BENCHMARK(BM_UserCounter)->ThreadPerCpu(); BENCHMARK_MAIN() diff --git a/test/cxx03_test.cc b/test/cxx03_test.cc index 56779d6602..9b2f63c18d 100644 --- a/test/cxx03_test.cc +++ b/test/cxx03_test.cc @@ -28,4 +28,13 @@ void BM_template1(benchmark::State& state) { BENCHMARK_TEMPLATE(BM_template1, long); BENCHMARK_TEMPLATE1(BM_template1, int); +void BM_counters(benchmark::State& state) { + while (state.KeepRunning()) { + volatile std::size_t x = state.iterations(); + ((void)x); + } + state.counters["Foo"] = 2; +} +BENCHMARK(BM_counters); + BENCHMARK_MAIN() diff --git a/test/reporter_output_test.cc b/test/reporter_output_test.cc index fc71f27cf2..4a77c2a303 100644 --- a/test/reporter_output_test.cc +++ b/test/reporter_output_test.cc @@ -283,6 +283,79 @@ ADD_CASES(&CSVOutputTests, { {"^\"BM_SummaryRepeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} }); + +// ========================================================================= // +// ------------------ Testing Custom Counter Output ------------------------ // +// ========================================================================= // + +// Test that non-aggregate data is printed by default +void BM_CustomCounters(benchmark::State& state) { + while (state.KeepRunning()) {} + switch (state.range(0)) { + case 0: + state.counters["foo"] = 0; + break; + case 1: + state.counters["bar"] = {42, benchmark::CT_Rate}; + break; + case 2: + state.counters["baz"] = {4, benchmark::CT_ThreadAverage}; + break; + case 3: + state.counters["boo"] = {8, benchmark::CT_ThreadAverageRate}; + break; + case 4: + state.counters["foo"] = 0; + state.counters["bar"] = {42, benchmark::CT_Rate}; + break; + default: + assert(false && "Test case not supported"); + } + // +} +BENCHMARK(BM_CustomCounters)->Arg(0)->Arg(1); +BENCHMARK(BM_CustomCounters)->Threads(4)->Arg(2)->Arg(3); +BENCHMARK(BM_CustomCounters)->Arg(4); +std::string ConsoleRE = "[0-9]{1,5} ns[ ]+[0-9]{1,5} ns[ ]+[0-9]+"; +ADD_CASES(&ConsoleOutputTests, { + {join("^BM_CustomCounters/0", ConsoleRE, "foo=0$")}, + {join("^BM_CustomCounters/1", ConsoleRE, "bar=" + dec_re + "/ns$")}, + {join("^BM_CustomCounters/2/threads:4", ConsoleRE, "baz=" + dec_re + "/thread$")}, + {join("^BM_CustomCounters/3/threads:4", ConsoleRE, "boo=" + dec_re + "/ns/thread$")}, + {join("^BM_CustomCounters/4", ConsoleRE, "((foo=0 bar=" + dec_re + "/ns)|(bar=" + dec_re + "/ns foo=0))$")} +}); +static int MakeJSONCase(std::string name, std::string cname, + std::string value, std::string type) +{ + AddCases(&JSONOutputTests, { + {"\"name\": \"" + name + "\",$"}, + {"\"counters\": \\["}, + {"\\{", MR_Next}, + {"\"name\": \"" + cname + "\",$", MR_Next}, + {"\"value\": " + value + ",$", MR_Next}, + {"\"type\": \"" + type + "\"$", MR_Next}, + {"}$", MR_Next}, + {"]$", MR_Next} + }); + return 0; +} +int json_test_anchor = ( + MakeJSONCase("BM_CustomCounters/0", "foo", "0", "unit") + , MakeJSONCase("BM_CustomCounters/1", "bar", dec_re, "unit/ns") + , MakeJSONCase("BM_CustomCounters/2/threads:4", "baz", dec_re, "unit/thread") + , MakeJSONCase("BM_CustomCounters/3/threads:4", "boo", dec_re, "unit/ns/thread") +); + +/* +ADD_CASES(&CSVOutputTests, { + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3_mean\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"}, + {"^\"BM_Repeat/repeats:3_stddev\",[0-9]+," + dec_re + "," + dec_re + ",ns,,,,,$"} +}); +*/ + // ========================================================================= // // --------------------------- TEST CASES END ------------------------------ // // ========================================================================= // @@ -309,7 +382,8 @@ int main(int argc, char* argv[]) { reporter.SetOutputStream(&out_stream); reporter.SetErrorStream(&err_stream); } - } TestCases[] = { + } + TestCases[] = { {"ConsoleReporter", ConsoleOutputTests, ConsoleErrorTests, CR}, {"JSONReporter", JSONOutputTests, JSONErrorTests, JR}, {"CSVReporter", CSVOutputTests, CSVErrorTests, CSVR} From 0f65ae4e94403e243bae40995f1617fe0f5a6899 Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Thu, 18 Aug 2016 01:36:06 -0600 Subject: [PATCH 2/3] fix invalid CHECK condition --- src/benchmark.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/benchmark.cc b/src/benchmark.cc index ea48bedb5c..67ea904d4c 100644 --- a/src/benchmark.cc +++ b/src/benchmark.cc @@ -801,7 +801,7 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b, std::pair res = total->counters.insert(KV); if (!res.second) { benchmark::Counter& dest = res.first->second; - CHECK_EQ(dest.Type(), KV.second.Type()) + CHECK_EQ(dest.type, KV.second.type) << "Cannot sum counters with different types"; dest += KV.second; } From 65538a181a8255d7d00826ddca0364a82e8a0cdb Mon Sep 17 00:00:00 2001 From: Eric Fiselier Date: Thu, 18 Aug 2016 04:38:03 -0600 Subject: [PATCH 3/3] work around GCC diagnostic --- src/counter.cc | 5 ++--- src/internal_macros.h | 7 +++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/counter.cc b/src/counter.cc index ccf258cc1a..eb8c823c5c 100644 --- a/src/counter.cc +++ b/src/counter.cc @@ -28,12 +28,11 @@ double Counter::FormatValue(double cpu_time, double num_threads) const { case CT_ThreadAverage: return value / num_threads; case CT_ThreadAverageRate: return (value / cpu_time) / num_threads; default: - assert(false && "unreachable"); + BENCHMARK_UNREACHABLE(); } } std::string Counter::FormatType(TimeUnit cpu_time_unit) const { - std::string str; std::string const unit_str = GetTimeUnitString(cpu_time_unit); switch (type) { case CT_Default: return ""; @@ -41,7 +40,7 @@ std::string Counter::FormatType(TimeUnit cpu_time_unit) const { case CT_ThreadAverage: return "/thread"; case CT_ThreadAverageRate: return "/" + unit_str + "/thread"; default: - assert(false && "unreachable"); + BENCHMARK_UNREACHABLE(); } } diff --git a/src/internal_macros.h b/src/internal_macros.h index 1080ac9435..9d3a55563b 100644 --- a/src/internal_macros.h +++ b/src/internal_macros.h @@ -2,6 +2,7 @@ #define BENCHMARK_INTERNAL_MACROS_H_ #include "benchmark/macros.h" +#include #ifndef __has_feature # define __has_feature(x) 0 @@ -15,6 +16,12 @@ # define BENCHMARK_NORETURN #endif +#if defined(__GNUC__) +# define BENCHMARK_UNREACHABLE() __builtin_unreachable() +#else +# define BENCHMARK_UNREACHABLE() assert(false && "unreachable") +#endif + #if defined(__CYGWIN__) # define BENCHMARK_OS_CYGWIN 1 #elif defined(_WIN32)