From 3bd55d0a9b1d3ee5d3166118c6fb3a90c49f990c Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 5 Nov 2025 14:43:23 +0100 Subject: [PATCH 1/4] fix(core): fix walltime stats computation to handle total round time as an input The computation is now in line with that codspeed-rust does, the interface is ready for other frameworks that may different iterations count from one round to another. --- core/include/codspeed.h | 9 +- core/src/walltime.cpp | 168 ++++++++++++++++++------------ google_benchmark/src/benchmark.cc | 34 +++--- 3 files changed, 118 insertions(+), 93 deletions(-) diff --git a/core/include/codspeed.h b/core/include/codspeed.h index e46184b..ba43982 100644 --- a/core/include/codspeed.h +++ b/core/include/codspeed.h @@ -34,11 +34,10 @@ class CodSpeed { struct RawWalltimeBenchmark { std::string name; std::string uri; - uint64_t iter_per_round; - double mean_ns; - double median_ns; - double stdev_ns; - std::vector round_times_ns; + // Number of iterations of each round + std::vector iters_per_round; + // Time taken by each round, meaning all its iterations, in nanoseconds + std::vector times_per_round_ns; }; void generate_codspeed_walltime_report( diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp index f347454..54e735b 100644 --- a/core/src/walltime.cpp +++ b/core/src/walltime.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -34,6 +35,27 @@ struct BenchmarkStats { uint64_t stdev_outlier_rounds; uint64_t iter_per_round; uint64_t warmup_iters; + + BenchmarkStats(double min_ns = 0.0, double max_ns = 0.0, double mean_ns = 0.0, + double stdev_ns = 0.0, double q1_ns = 0.0, + double median_ns = 0.0, double q3_ns = 0.0, + uint64_t rounds = 0, double total_time = 0.0, + uint64_t iqr_outlier_rounds = 0, + uint64_t stdev_outlier_rounds = 0, uint64_t iter_per_round = 0, + uint64_t warmup_iters = 0) + : min_ns(min_ns), + max_ns(max_ns), + mean_ns(mean_ns), + stdev_ns(stdev_ns), + q1_ns(q1_ns), + median_ns(median_ns), + q3_ns(q3_ns), + rounds(rounds), + total_time(total_time), + iqr_outlier_rounds(iqr_outlier_rounds), + stdev_outlier_rounds(stdev_outlier_rounds), + iter_per_round(iter_per_round), + warmup_iters(warmup_iters) {} }; struct BenchmarkMetadata { @@ -46,8 +68,9 @@ struct CodspeedWalltimeBenchmark { BenchmarkStats stats; }; -double compute_quantile(const std::vector &data, double quantile) { - size_t n = data.size(); +static double compute_quantile(const std::vector &sorted_data, + double quantile) { + size_t n = sorted_data.size(); if (n == 0) return 0.0; double pos = quantile * (n - 1); @@ -55,42 +78,12 @@ double compute_quantile(const std::vector &data, double quantile) { double d = pos - k; if (k + 1 < n) { - return data[k] + d * (data[k + 1] - data[k]); + return sorted_data[k] + d * (sorted_data[k + 1] - sorted_data[k]); } - return data[k]; -} - -void compute_iqr_and_outliers(const std::vector ×_ns, double mean, - double stdev, double &q1, double &q3, double &iqr, - size_t &iqr_outlier_rounds, - size_t &stdev_outlier_rounds) { - std::vector sorted_times = times_ns; - std::sort(sorted_times.begin(), sorted_times.end()); - - q1 = compute_quantile(sorted_times, 0.25); - q3 = compute_quantile(sorted_times, 0.75); - - iqr = q3 - q1; - - const double IQR_OUTLIER_FACTOR = 1.5; - const double STDEV_OUTLIER_FACTOR = 3.0; - - iqr_outlier_rounds = - std::count_if(sorted_times.begin(), sorted_times.end(), - [q1, q3, iqr, IQR_OUTLIER_FACTOR](double x) { - return x < q1 - IQR_OUTLIER_FACTOR * iqr || - x > q3 + IQR_OUTLIER_FACTOR * iqr; - }); - - stdev_outlier_rounds = - std::count_if(sorted_times.begin(), sorted_times.end(), - [mean, stdev, STDEV_OUTLIER_FACTOR](double x) { - return x < mean - STDEV_OUTLIER_FACTOR * stdev || - x > mean + STDEV_OUTLIER_FACTOR * stdev; - }); + return sorted_data[k]; } -std::string escapeBackslashes(const std::string &input) { +static std::string escape_backslashes(const std::string &input) { std::string output; for (char c : input) { if (c == '\\') { @@ -102,7 +95,7 @@ std::string escapeBackslashes(const std::string &input) { return output; } -void write_codspeed_benchmarks_to_json( +static void write_codspeed_benchmarks_to_json( const std::vector &benchmarks) { std::ostringstream oss; @@ -132,8 +125,8 @@ void write_codspeed_benchmarks_to_json( const auto &metadata = benchmark.metadata; oss << " {\n"; - oss << " \"name\": \"" << escapeBackslashes(metadata.name) << "\",\n"; - oss << " \"uri\": \"" << escapeBackslashes(metadata.uri) << "\",\n"; + oss << " \"name\": \"" << escape_backslashes(metadata.name) << "\",\n"; + oss << " \"uri\": \"" << escape_backslashes(metadata.uri) << "\",\n"; // TODO: Manage config fields from actual configuration oss << " \"config\": {\n"; oss << " \"warmup_time_ns\": null,\n"; @@ -202,40 +195,83 @@ void generate_codspeed_walltime_report( std::vector codspeed_walltime_benchmarks; for (const auto &raw_benchmark : raw_walltime_benchmarks) { - CodspeedWalltimeBenchmark codspeed_benchmark; - codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri}; + assert(raw_benchmark.iters_per_round.size() == + raw_benchmark.times_per_round_ns.size()); + + assert(raw_benchmark.iters_per_round.size() != 0); + + // Convert total round times to per-iteration times + std::vector per_iteration_times_ns; + for (size_t i = 0; i < raw_benchmark.times_per_round_ns.size(); i++) { + assert(raw_benchmark.iters_per_round[i] != 0); + double per_iter_time_ns = raw_benchmark.times_per_round_ns[i] / + raw_benchmark.iters_per_round[i]; + per_iteration_times_ns.push_back(per_iter_time_ns); + } + // Sort for quantile computation + std::vector sorted_per_iter_times_ns = per_iteration_times_ns; + std::sort(sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end()); + + // Compute statistics from per-iteration times + double mean_ns = std::accumulate(per_iteration_times_ns.begin(), + per_iteration_times_ns.end(), 0.0) / + per_iteration_times_ns.size(); + + double variance = 0.0; + for (double time_ns : per_iteration_times_ns) { + double diff = time_ns - mean_ns; + variance += diff * diff; + } + double stdev_ns = std::sqrt(variance / per_iteration_times_ns.size()); + const double STDEV_OUTLIER_FACTOR = 3.0; + size_t stdev_outlier_rounds = std::count_if( + sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), + [mean_ns, stdev_ns, STDEV_OUTLIER_FACTOR](double x) { + return x < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns || + x > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns; + }); + + double q1_ns = compute_quantile(sorted_per_iter_times_ns, 0.25); + double median_ns = compute_quantile(sorted_per_iter_times_ns, 0.5); + double q3_ns = compute_quantile(sorted_per_iter_times_ns, 0.75); + + double iqr_ns = q3_ns - q1_ns; + const double IQR_OUTLIER_FACTOR = 1.5; + size_t iqr_outlier_rounds = std::count_if( + sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), + [q1_ns, q3_ns, iqr_ns, IQR_OUTLIER_FACTOR](double x) { + return x < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || + x > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns; + }); + + // Compute total time in seconds double total_time = - std::accumulate(raw_benchmark.round_times_ns.begin(), - raw_benchmark.round_times_ns.end(), 0.0) / + std::accumulate(raw_benchmark.times_per_round_ns.begin(), + raw_benchmark.times_per_round_ns.end(), 0.0) / 1e9; - double mean = raw_benchmark.mean_ns; - double median = raw_benchmark.median_ns; - double stdev = raw_benchmark.stdev_ns; - double q1, q3, iqr; - size_t iqr_outlier_rounds, stdev_outlier_rounds; - compute_iqr_and_outliers(raw_benchmark.round_times_ns, mean, stdev, q1, q3, - iqr, iqr_outlier_rounds, stdev_outlier_rounds); + // TODO: CodSpeed format only supports one iter_per_round for all rounds, + // for now take the average + uint64_t avg_iters_per_round = + std::accumulate(raw_benchmark.iters_per_round.begin(), + raw_benchmark.iters_per_round.end(), 0ULL) / + raw_benchmark.iters_per_round.size(); // Populate stats - codspeed_benchmark.stats = { - *std::min_element(raw_benchmark.round_times_ns.begin(), - raw_benchmark.round_times_ns.end()), - *std::max_element(raw_benchmark.round_times_ns.begin(), - raw_benchmark.round_times_ns.end()), - mean, - stdev, - q1, - median, - q3, - raw_benchmark.round_times_ns.size(), - total_time, - iqr_outlier_rounds, - stdev_outlier_rounds, - raw_benchmark.iter_per_round, - 0 // TODO: warmup_iters - }; + BenchmarkStats stats(*std::min_element(sorted_per_iter_times_ns.begin(), + sorted_per_iter_times_ns.end()), + *std::max_element(sorted_per_iter_times_ns.begin(), + sorted_per_iter_times_ns.end()), + mean_ns, stdev_ns, q1_ns, median_ns, q3_ns, + raw_benchmark.times_per_round_ns.size(), total_time, + iqr_outlier_rounds, stdev_outlier_rounds, + avg_iters_per_round, + 0 // TODO: warmup_iters + ); + CodspeedWalltimeBenchmark codspeed_benchmark; + codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri}; + codspeed_benchmark.stats = stats; codspeed_walltime_benchmarks.push_back(codspeed_benchmark); } diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index 6452d89..8e4c2c2 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -382,6 +382,8 @@ void FlushStreams(BenchmarkReporter* reporter) { } #ifdef CODSPEED_WALLTIME +// NOTE: What Google Benchmark calls a repetition, Codspeed calls a round. +// // We use real time by default, but we could offer CPU time usage as a build // option, open an issue if you need it. codspeed::RawWalltimeBenchmark generate_raw_walltime_data( @@ -402,27 +404,15 @@ codspeed::RawWalltimeBenchmark generate_raw_walltime_data( walltime_data.uri = "unknown_file::" + walltime_data.uri; } - walltime_data.iter_per_round = run.iterations; - walltime_data.round_times_ns.push_back(run.GetAdjustedRealTime()); - } + // Collect iteration count for this round/repetition + walltime_data.iters_per_round.push_back( + static_cast(run.iterations)); - if (run_results.aggregates_only.empty()) { - // If run has no aggreagates, it means that only one round was performed. - // Use this time as a mean, median, and set stdev to 0. - double only_round_time_ns = walltime_data.round_times_ns[0]; - walltime_data.mean_ns = only_round_time_ns; - walltime_data.median_ns = only_round_time_ns; - walltime_data.stdev_ns = 0; - } else { - for (const auto& aggregate_run : run_results.aggregates_only) { - if (aggregate_run.aggregate_name == "mean") { - walltime_data.mean_ns = aggregate_run.GetAdjustedRealTime(); - } else if (aggregate_run.aggregate_name == "median") { - walltime_data.median_ns = aggregate_run.GetAdjustedRealTime(); - } else if (aggregate_run.aggregate_name == "stddev") { - walltime_data.stdev_ns = aggregate_run.GetAdjustedRealTime(); - } - } + // Collect total round time in nanoseconds + // real_accumulated_time is in seconds, convert to nanoseconds + double round_time_ns = + run.real_accumulated_time * GetTimeUnitMultiplier(kNanosecond); + walltime_data.times_per_round_ns.push_back(round_time_ns); } return walltime_data; @@ -568,12 +558,12 @@ void RunBenchmarks(const std::vector& benchmarks, internal::BenchmarkRunner& runner = runners[repetition_index]; #ifdef CODSPEED_WALLTIME - auto codspeed = codspeed::CodSpeed::getInstance(); + auto* codspeed = codspeed::CodSpeed::getInstance(); if (codspeed != nullptr) { codspeed->start_benchmark(runner.GetBenchmarkName()); } #endif - + runner.DoOneRepetition(); if (runner.HasRepeatsRemaining()) { continue; From 6ecade853c3bd5bf59317f67f5b95605a3ce1b6f Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 5 Nov 2025 14:49:04 +0100 Subject: [PATCH 2/4] feat(google_benchmark): increase default codspeed walltime repetitions This allows codspeed to have more rounds and improves default statistics, which would just be computed as 1 single element in the boxplot since we only have access to round times. --- google_benchmark/src/benchmark.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/google_benchmark/src/benchmark.cc b/google_benchmark/src/benchmark.cc index 8e4c2c2..9937e5c 100644 --- a/google_benchmark/src/benchmark.cc +++ b/google_benchmark/src/benchmark.cc @@ -94,7 +94,14 @@ BM_DEFINE_double(benchmark_min_warmup_time, 0.0); // The number of runs of each benchmark. If greater than 1, the mean and // standard deviation of the runs will be reported. +// CODSPEED: change default repetitions during walltime runs in order to have +// better statistics by default +// https://github.com/google/benchmark/blob/main/docs/user_guide.md#reporting-statistics +#ifdef CODSPEED_WALLTIME +BM_DEFINE_int32(benchmark_repetitions, 5); +#else BM_DEFINE_int32(benchmark_repetitions, 1); +#endif // If enabled, forces each benchmark to execute exactly one iteration and one // repetition, bypassing any configured From ccded65fc0d5b4ff3e3e98d37052a3d83dc3936b Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Wed, 5 Nov 2025 14:55:51 +0100 Subject: [PATCH 3/4] ci: fix incorrect step name in bazel integration test --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2dfd6f3..e708577 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: # Share repository cache between workflows. repository-cache: true - - name: Build and run benchmarks + - name: Build benchmark example run: | bazel build //examples/google_benchmark_bazel:my_benchmark --//core:codspeed_mode=${{ matrix.codspeed-mode }} --//core:strict_warnings=on From da312ec3ceda1a4e0a4d9b1cd232cc4097d63a9c Mon Sep 17 00:00:00 2001 From: Guillaume Lagrange Date: Thu, 6 Nov 2025 11:15:43 +0100 Subject: [PATCH 4/4] feat(core): add unit tests for statistics computation --- Makefile | 5 + core/src/walltime.cpp | 193 ++++++++++++++++----------------------- core/src/walltime.h | 55 +++++++++++ core/test/CMakeLists.txt | 3 + core/test/walltime.cpp | 181 ++++++++++++++++++++++++++++++++++++ 5 files changed, 325 insertions(+), 112 deletions(-) create mode 100644 Makefile create mode 100644 core/src/walltime.h create mode 100644 core/test/walltime.cpp diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..353e9cd --- /dev/null +++ b/Makefile @@ -0,0 +1,5 @@ +.PHONY: format +format: + @echo "Formatting C++ files..." + @git ls-files '*.cpp' '*.h' | grep -v '^google_benchmark/' | xargs -r clang-format -i + @echo "Formatting complete!" diff --git a/core/src/walltime.cpp b/core/src/walltime.cpp index 54e735b..73b4459 100644 --- a/core/src/walltime.cpp +++ b/core/src/walltime.cpp @@ -1,3 +1,5 @@ +#include "walltime.h" + #include #include #include @@ -20,44 +22,6 @@ namespace codspeed { -// Times are per iteration -struct BenchmarkStats { - double min_ns; - double max_ns; - double mean_ns; - double stdev_ns; - double q1_ns; - double median_ns; - double q3_ns; - uint64_t rounds; - double total_time; - uint64_t iqr_outlier_rounds; - uint64_t stdev_outlier_rounds; - uint64_t iter_per_round; - uint64_t warmup_iters; - - BenchmarkStats(double min_ns = 0.0, double max_ns = 0.0, double mean_ns = 0.0, - double stdev_ns = 0.0, double q1_ns = 0.0, - double median_ns = 0.0, double q3_ns = 0.0, - uint64_t rounds = 0, double total_time = 0.0, - uint64_t iqr_outlier_rounds = 0, - uint64_t stdev_outlier_rounds = 0, uint64_t iter_per_round = 0, - uint64_t warmup_iters = 0) - : min_ns(min_ns), - max_ns(max_ns), - mean_ns(mean_ns), - stdev_ns(stdev_ns), - q1_ns(q1_ns), - median_ns(median_ns), - q3_ns(q3_ns), - rounds(rounds), - total_time(total_time), - iqr_outlier_rounds(iqr_outlier_rounds), - stdev_outlier_rounds(stdev_outlier_rounds), - iter_per_round(iter_per_round), - warmup_iters(warmup_iters) {} -}; - struct BenchmarkMetadata { std::string name; std::string uri; @@ -190,85 +154,90 @@ static void write_codspeed_benchmarks_to_json( } } +BenchmarkStats compute_benchmark_stats( + const RawWalltimeBenchmark &raw_benchmark) { + assert(raw_benchmark.iters_per_round.size() == + raw_benchmark.times_per_round_ns.size()); + + assert(raw_benchmark.iters_per_round.size() != 0); + + // Convert total round times to per-iteration times + std::vector per_iteration_times_ns; + for (size_t i = 0; i < raw_benchmark.times_per_round_ns.size(); i++) { + assert(raw_benchmark.iters_per_round[i] != 0); + double per_iter_time_ns = + raw_benchmark.times_per_round_ns[i] / raw_benchmark.iters_per_round[i]; + per_iteration_times_ns.push_back(per_iter_time_ns); + } + + // Sort for quantile computation + std::vector sorted_per_iter_times_ns = per_iteration_times_ns; + std::sort(sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end()); + + // Compute statistics from per-iteration times + double mean_ns = std::accumulate(per_iteration_times_ns.begin(), + per_iteration_times_ns.end(), 0.0) / + per_iteration_times_ns.size(); + + double variance = 0.0; + for (double time_ns : per_iteration_times_ns) { + double diff = time_ns - mean_ns; + variance += diff * diff; + } + double stdev_ns = std::sqrt(variance / per_iteration_times_ns.size()); + const double STDEV_OUTLIER_FACTOR = 3.0; + size_t stdev_outlier_rounds = std::count_if( + sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), + [mean_ns, stdev_ns, STDEV_OUTLIER_FACTOR](double x) { + return x < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns || + x > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns; + }); + + double q1_ns = compute_quantile(sorted_per_iter_times_ns, 0.25); + double median_ns = compute_quantile(sorted_per_iter_times_ns, 0.5); + double q3_ns = compute_quantile(sorted_per_iter_times_ns, 0.75); + + double iqr_ns = q3_ns - q1_ns; + const double IQR_OUTLIER_FACTOR = 1.5; + size_t iqr_outlier_rounds = std::count_if( + sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), + [q1_ns, q3_ns, iqr_ns, IQR_OUTLIER_FACTOR](double x) { + return x < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || + x > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns; + }); + + // Compute total time in seconds + double total_time = + std::accumulate(raw_benchmark.times_per_round_ns.begin(), + raw_benchmark.times_per_round_ns.end(), 0.0) / + 1e9; + + // TODO: CodSpeed format only supports one iter_per_round for all rounds, + // for now take the average + uint64_t avg_iters_per_round = + std::accumulate(raw_benchmark.iters_per_round.begin(), + raw_benchmark.iters_per_round.end(), 0ULL) / + raw_benchmark.iters_per_round.size(); + + // Populate stats + return BenchmarkStats(*std::min_element(sorted_per_iter_times_ns.begin(), + sorted_per_iter_times_ns.end()), + *std::max_element(sorted_per_iter_times_ns.begin(), + sorted_per_iter_times_ns.end()), + mean_ns, stdev_ns, q1_ns, median_ns, q3_ns, + raw_benchmark.times_per_round_ns.size(), total_time, + iqr_outlier_rounds, stdev_outlier_rounds, + avg_iters_per_round, + 0 // TODO: warmup_iters + ); +} + void generate_codspeed_walltime_report( const std::vector &raw_walltime_benchmarks) { std::vector codspeed_walltime_benchmarks; for (const auto &raw_benchmark : raw_walltime_benchmarks) { - assert(raw_benchmark.iters_per_round.size() == - raw_benchmark.times_per_round_ns.size()); - - assert(raw_benchmark.iters_per_round.size() != 0); - - // Convert total round times to per-iteration times - std::vector per_iteration_times_ns; - for (size_t i = 0; i < raw_benchmark.times_per_round_ns.size(); i++) { - assert(raw_benchmark.iters_per_round[i] != 0); - double per_iter_time_ns = raw_benchmark.times_per_round_ns[i] / - raw_benchmark.iters_per_round[i]; - per_iteration_times_ns.push_back(per_iter_time_ns); - } - - // Sort for quantile computation - std::vector sorted_per_iter_times_ns = per_iteration_times_ns; - std::sort(sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end()); - - // Compute statistics from per-iteration times - double mean_ns = std::accumulate(per_iteration_times_ns.begin(), - per_iteration_times_ns.end(), 0.0) / - per_iteration_times_ns.size(); - - double variance = 0.0; - for (double time_ns : per_iteration_times_ns) { - double diff = time_ns - mean_ns; - variance += diff * diff; - } - double stdev_ns = std::sqrt(variance / per_iteration_times_ns.size()); - const double STDEV_OUTLIER_FACTOR = 3.0; - size_t stdev_outlier_rounds = std::count_if( - sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), - [mean_ns, stdev_ns, STDEV_OUTLIER_FACTOR](double x) { - return x < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns || - x > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns; - }); - - double q1_ns = compute_quantile(sorted_per_iter_times_ns, 0.25); - double median_ns = compute_quantile(sorted_per_iter_times_ns, 0.5); - double q3_ns = compute_quantile(sorted_per_iter_times_ns, 0.75); - - double iqr_ns = q3_ns - q1_ns; - const double IQR_OUTLIER_FACTOR = 1.5; - size_t iqr_outlier_rounds = std::count_if( - sorted_per_iter_times_ns.begin(), sorted_per_iter_times_ns.end(), - [q1_ns, q3_ns, iqr_ns, IQR_OUTLIER_FACTOR](double x) { - return x < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || - x > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns; - }); - - // Compute total time in seconds - double total_time = - std::accumulate(raw_benchmark.times_per_round_ns.begin(), - raw_benchmark.times_per_round_ns.end(), 0.0) / - 1e9; - - // TODO: CodSpeed format only supports one iter_per_round for all rounds, - // for now take the average - uint64_t avg_iters_per_round = - std::accumulate(raw_benchmark.iters_per_round.begin(), - raw_benchmark.iters_per_round.end(), 0ULL) / - raw_benchmark.iters_per_round.size(); - - // Populate stats - BenchmarkStats stats(*std::min_element(sorted_per_iter_times_ns.begin(), - sorted_per_iter_times_ns.end()), - *std::max_element(sorted_per_iter_times_ns.begin(), - sorted_per_iter_times_ns.end()), - mean_ns, stdev_ns, q1_ns, median_ns, q3_ns, - raw_benchmark.times_per_round_ns.size(), total_time, - iqr_outlier_rounds, stdev_outlier_rounds, - avg_iters_per_round, - 0 // TODO: warmup_iters - ); + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); CodspeedWalltimeBenchmark codspeed_benchmark; codspeed_benchmark.metadata = {raw_benchmark.name, raw_benchmark.uri}; codspeed_benchmark.stats = stats; diff --git a/core/src/walltime.h b/core/src/walltime.h new file mode 100644 index 0000000..fad6658 --- /dev/null +++ b/core/src/walltime.h @@ -0,0 +1,55 @@ +#ifndef CODSPEED_WALLTIME_INTERNAL_H +#define CODSPEED_WALLTIME_INTERNAL_H + +#include +#include + +#include "codspeed.h" + +namespace codspeed { + +// Times are per iteration +struct BenchmarkStats { + double min_ns; + double max_ns; + double mean_ns; + double stdev_ns; + double q1_ns; + double median_ns; + double q3_ns; + uint64_t rounds; + double total_time; + uint64_t iqr_outlier_rounds; + uint64_t stdev_outlier_rounds; + uint64_t iter_per_round; + uint64_t warmup_iters; + + BenchmarkStats(double min_ns = 0.0, double max_ns = 0.0, double mean_ns = 0.0, + double stdev_ns = 0.0, double q1_ns = 0.0, + double median_ns = 0.0, double q3_ns = 0.0, + uint64_t rounds = 0, double total_time = 0.0, + uint64_t iqr_outlier_rounds = 0, + uint64_t stdev_outlier_rounds = 0, uint64_t iter_per_round = 0, + uint64_t warmup_iters = 0) + : min_ns(min_ns), + max_ns(max_ns), + mean_ns(mean_ns), + stdev_ns(stdev_ns), + q1_ns(q1_ns), + median_ns(median_ns), + q3_ns(q3_ns), + rounds(rounds), + total_time(total_time), + iqr_outlier_rounds(iqr_outlier_rounds), + stdev_outlier_rounds(stdev_outlier_rounds), + iter_per_round(iter_per_round), + warmup_iters(warmup_iters) {} +}; + +// Compute benchmark statistics from raw walltime data +BenchmarkStats compute_benchmark_stats( + const RawWalltimeBenchmark &raw_benchmark); + +} // namespace codspeed + +#endif // CODSPEED_WALLTIME_INTERNAL_H diff --git a/core/test/CMakeLists.txt b/core/test/CMakeLists.txt index 9e2f220..9266e20 100644 --- a/core/test/CMakeLists.txt +++ b/core/test/CMakeLists.txt @@ -9,8 +9,11 @@ FetchContent_MakeAvailable(googletest) add_executable(unit_tests uri.cpp codspeed.cpp + walltime.cpp ) +target_include_directories(unit_tests PRIVATE ${CMAKE_SOURCE_DIR}/src) + target_link_libraries(unit_tests PRIVATE codspeed diff --git a/core/test/walltime.cpp b/core/test/walltime.cpp new file mode 100644 index 0000000..717b958 --- /dev/null +++ b/core/test/walltime.cpp @@ -0,0 +1,181 @@ +#include "walltime.h" + +#include + +#include + +namespace codspeed { + +TEST(WalltimeStatsTest, BasicStatsComputation) { + // Test with a simple benchmark with consistent iterations + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "test_benchmark"; + raw_benchmark.uri = "test_file.cpp"; + + // 5 rounds, each with 10 iterations + // Total round times: 1000ns, 2000ns, 3000ns, 4000ns, 6000ns + // Per-iteration times: 100ns, 200ns, 300ns, 400ns, 600ns + // This creates a right-skewed distribution where mean != median + std::vector round_times = { + 6000.0, 3000.0, 1000.0, 2000.0, 4000.0, + }; + for (double time : round_times) { + raw_benchmark.iters_per_round.push_back(10); + raw_benchmark.times_per_round_ns.push_back(time); + } + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + // Per-iteration times are: 100, 200, 300, 400, 600 + EXPECT_DOUBLE_EQ(stats.min_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.max_ns, 600.0); + EXPECT_DOUBLE_EQ(stats.mean_ns, 320.0); // (100+200+300+400+600)/5 + EXPECT_DOUBLE_EQ(stats.median_ns, + 300.0); // Middle value (different from mean) + EXPECT_DOUBLE_EQ(stats.q1_ns, 200.0); + EXPECT_DOUBLE_EQ(stats.q3_ns, 400.0); + EXPECT_EQ(stats.rounds, 5); + + // Total time: (1000+2000+3000+4000+6000)ns = 16000ns = 0.000016s + EXPECT_DOUBLE_EQ(stats.total_time, 16000.0 / 1e9); + + // Average iterations per round: (10+10+10+10+10)/5 = 10 + EXPECT_EQ(stats.iter_per_round, 10); + + // Standard deviation: sqrt(variance) + // variance = [(100-320)^2 + (200-320)^2 + (300-320)^2 + (400-320)^2 + + // (600-320)^2] / 5 + // = [48400 + 14400 + 400 + 6400 + 78400] / 5 = 29600 + // stdev = sqrt(29600) ≈ 172.047 + double expected_stdev = std::sqrt(29600.0); + EXPECT_NEAR(stats.stdev_ns, expected_stdev, 0.01); + + // No outliers in this right-skewed data (600 is not extreme enough) + EXPECT_EQ(stats.iqr_outlier_rounds, 0); + EXPECT_EQ(stats.stdev_outlier_rounds, 0); +} + +TEST(WalltimeStatsTest, StdevOutlierDetection) { + // Test standard deviation outlier detection (3-sigma rule) + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "stdev_outlier_benchmark"; + raw_benchmark.uri = "test_file.cpp"; + + // Create data with a clear outlier + std::vector round_times = {1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 1.0, 1.0, 50.0}; + for (double time : round_times) { + raw_benchmark.iters_per_round.push_back(1); + raw_benchmark.times_per_round_ns.push_back(time); + } + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + EXPECT_NEAR(stats.mean_ns, 3.8823529, 0.001); + + EXPECT_DOUBLE_EQ(stats.median_ns, 1.0); + + EXPECT_NEAR(stats.stdev_ns, 11.53, 0.001); + + EXPECT_EQ(stats.stdev_outlier_rounds, 1); +} + +TEST(WalltimeStatsTest, IQROutlierDetection) { + // Test IQR outlier detection (1.5*IQR rule) + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "iqr_outlier_benchmark"; + raw_benchmark.uri = "test_file.cpp"; + + // Create data with a clear outlier + // Per-iteration times: 100, 110, 120, 130, 140, 500 (ns) + std::vector values = {100, 110, 120, 130, 140, 500}; + for (double val : values) { + raw_benchmark.iters_per_round.push_back(1); + raw_benchmark.times_per_round_ns.push_back(val); + } + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + EXPECT_EQ(stats.q1_ns, 112.5); + EXPECT_EQ(stats.q3_ns, 137.5); + // IQR = 137.5 - 112.5 = 25 + // Lower bound: 112.5 - 1.5*25 = 75 + // Upper bound: 137.5 + 1.5*25 = 175 + // Outlier: 500 (> 175) + EXPECT_EQ(stats.iqr_outlier_rounds, 1); +} + +TEST(WalltimeStatsTest, SingleRound) { + // Test with a single round + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "single_round_benchmark"; + raw_benchmark.uri = "test_file.cpp"; + + raw_benchmark.iters_per_round.push_back(5); + raw_benchmark.times_per_round_ns.push_back(500.0); // 100ns per iter + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + // With a single value, all stats should be the same + EXPECT_DOUBLE_EQ(stats.min_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.max_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.mean_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.median_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.q1_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.q3_ns, 100.0); + EXPECT_DOUBLE_EQ(stats.stdev_ns, 0.0); + EXPECT_EQ(stats.rounds, 1); + EXPECT_EQ(stats.iqr_outlier_rounds, 0); + EXPECT_EQ(stats.stdev_outlier_rounds, 0); + + // Total time: 500ns = 0.0000005s + EXPECT_DOUBLE_EQ(stats.total_time, 500.0 / 1e9); +} + +TEST(WalltimeStatsTest, QantileComputation) { + // Test quantile computation with a specific dataset + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "quantile_test"; + raw_benchmark.uri = "test_file.cpp"; + + // Per-iteration times: 10, 20, 30, 40, 50, 60, 70, 80, 90 (ns) + std::vector round_times = {10.0, 20.0, 30.0, 40.0, 50.0, + 60.0, 70.0, 80.0, 90.0}; + for (double time : round_times) { + raw_benchmark.iters_per_round.push_back(1); + raw_benchmark.times_per_round_ns.push_back(time); + } + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + // With 9 values (indices 0-8): + // Q1 (0.25): pos = 0.25 * 8 = 2.0 -> value at index 2 = 30 + // Median (0.5): pos = 0.5 * 8 = 4.0 -> value at index 4 = 50 + // Q3 (0.75): pos = 0.75 * 8 = 6.0 -> value at index 6 = 70 + EXPECT_DOUBLE_EQ(stats.q1_ns, 30.0); + EXPECT_DOUBLE_EQ(stats.median_ns, 50.0); + EXPECT_DOUBLE_EQ(stats.q3_ns, 70.0); +} + +TEST(WalltimeStatsTest, QuantileInterpolation) { + // Test quantile computation with a specific dataset + RawWalltimeBenchmark raw_benchmark; + raw_benchmark.name = "quantile_test"; + raw_benchmark.uri = "test_file.cpp"; + + std::vector round_times = {10.0, 20.0, 30.0, 40.0, + 50.0, 60.0, 70.0, 80.0}; + for (double time : round_times) { + raw_benchmark.iters_per_round.push_back(1); + raw_benchmark.times_per_round_ns.push_back(time); + } + + BenchmarkStats stats = compute_benchmark_stats(raw_benchmark); + + EXPECT_DOUBLE_EQ(stats.q1_ns, 27.5); + EXPECT_DOUBLE_EQ(stats.median_ns, 45.0); + EXPECT_DOUBLE_EQ(stats.q3_ns, 62.5); +} + +} // namespace codspeed