[llvm-exegesis] Add option to specify the number of measurement repetitions

[boomanaiden154]

Currently, the llvm-exegesis LatencyBenchmarkRunner repeats the benchmark several times (currently 30) and then aggregates the result to deal with noise in the measurement process. With this patch, the number of repetitions to perform is made configurable rather than left as a static number. This allows for significantly faster execution in situations where someone is performing a task like experimenting with memory annotations where the exact cycle counts might not be useful, and also allows for increased precision when desired.

[llvmbot]

@llvm/pr-subscribers-tools-llvm-exegesis

Author: Aiden Grossman (boomanaiden154)

Changes

Currently, the llvm-exegesis LatencyBenchmarkRunner repeats the benchmark several times (currently 30) and then aggregates the result to deal with noise in the measurement process. With this patch, the number of repetitions to perform is made configurable rather than left as a static number. This allows for significantly faster execution in situations where someone is performing a task like experimenting with memory annotations where the exact cycle counts might not be useful, and also allows for increased precision when desired.

---

Full diff: https://github.com/llvm/llvm-project/pull/74276.diff

5 Files Affected:

• (modified) llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp (+4-3)
• (modified) llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h (+3-1)
• (modified) llvm/tools/llvm-exegesis/lib/Target.cpp (+8-4)
• (modified) llvm/tools/llvm-exegesis/lib/Target.h (+4-2)
• (modified) llvm/tools/llvm-exegesis/llvm-exegesis.cpp (+6)

diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
index 753efed138163..7e560c2d64f27 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp
@@ -21,11 +21,13 @@ namespace exegesis {
 LatencyBenchmarkRunner::LatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
-    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode)
+    Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode,
+    unsigned BenchmarkRepeatCount)
     : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) {
   assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) &&
          "invalid mode");
   ResultAggMode = ResultAgg;
+  BenchmarkRepetitions = BenchmarkRepeatCount;
 }
 
 LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default;
@@ -68,13 +70,12 @@ Expected<std::vector<BenchmarkMeasure>> LatencyBenchmarkRunner::runMeasurements(
   // Cycle measurements include some overhead from the kernel. Repeat the
   // measure several times and return the aggregated value, as specified by
   // ResultAggMode.
-  constexpr const int NumMeasurements = 30;
   llvm::SmallVector<int64_t, 4> AccumulatedValues;
   double MinVariance = std::numeric_limits<double>::infinity();
   const char *CounterName = State.getPfmCounters().CycleCounter;
   // Values count for each run.
   int ValuesCount = 0;
-  for (size_t I = 0; I < NumMeasurements; ++I) {
+  for (size_t I = 0; I < BenchmarkRepetitions; ++I) {
     auto ExpectedCounterValues = Executor.runAndSample(CounterName);
     if (!ExpectedCounterValues)
       return ExpectedCounterValues.takeError();
diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
index 34b912f0abded..02dedeca63850 100644
--- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
+++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h
@@ -24,7 +24,8 @@ class LatencyBenchmarkRunner : public BenchmarkRunner {
   LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode,
                          BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
                          Benchmark::ResultAggregationModeE ResultAggMode,
-                         ExecutionModeE ExecutionMode);
+                         ExecutionModeE ExecutionMode,
+                         unsigned BenchmarkRepeatCount);
   ~LatencyBenchmarkRunner() override;
 
 private:
@@ -32,6 +33,7 @@ class LatencyBenchmarkRunner : public BenchmarkRunner {
   runMeasurements(const FunctionExecutor &Executor) const override;
 
   Benchmark::ResultAggregationModeE ResultAggMode;
+  unsigned BenchmarkRepetitions;
 };
 } // namespace exegesis
 } // namespace llvm
diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp
index aed093548f158..67eddb9dbb1db 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Target.cpp
@@ -79,7 +79,8 @@ ExegesisTarget::createBenchmarkRunner(
     Benchmark::ModeE Mode, const LLVMState &State,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
     BenchmarkRunner::ExecutionModeE ExecutionMode,
-    Benchmark::ResultAggregationModeE ResultAggMode) const {
+    Benchmark::ResultAggregationModeE ResultAggMode,
+    unsigned BenchmarkRepeatCount) const {
   PfmCountersInfo PfmCounters = State.getPfmCounters();
   switch (Mode) {
   case Benchmark::Unknown:
@@ -101,7 +102,8 @@ ExegesisTarget::createBenchmarkRunner(
                   "the kernel for real event counts."));
     }
     return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector,
-                                        ResultAggMode, ExecutionMode);
+                                        ResultAggMode, ExecutionMode,
+                                        BenchmarkRepeatCount);
   case Benchmark::Uops:
     if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure &&
         !PfmCounters.UopsCounter && !PfmCounters.IssueCounters)
@@ -130,9 +132,11 @@ std::unique_ptr<BenchmarkRunner> ExegesisTarget::createLatencyBenchmarkRunner(
     const LLVMState &State, Benchmark::ModeE Mode,
     BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
     Benchmark::ResultAggregationModeE ResultAggMode,
-    BenchmarkRunner::ExecutionModeE ExecutionMode) const {
+    BenchmarkRunner::ExecutionModeE ExecutionMode,
+    unsigned BenchmarkRepeatCount) const {
   return std::make_unique<LatencyBenchmarkRunner>(
-      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode);
+      State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode,
+      BenchmarkRepeatCount);
 }
 
 std::unique_ptr<BenchmarkRunner> ExegesisTarget::createUopsBenchmarkRunner(
diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h
index 6de5b3c1065f1..26f5f5403aa44 100644
--- a/llvm/tools/llvm-exegesis/lib/Target.h
+++ b/llvm/tools/llvm-exegesis/lib/Target.h
@@ -262,7 +262,8 @@ class ExegesisTarget {
       Benchmark::ModeE Mode, const LLVMState &State,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       BenchmarkRunner::ExecutionModeE ExecutionMode,
-      Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const;
+      Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min,
+      unsigned BenchmarkRepeatCount = 30) const;
 
   // Returns the ExegesisTarget for the given triple or nullptr if the target
   // does not exist.
@@ -305,7 +306,8 @@ class ExegesisTarget {
       const LLVMState &State, Benchmark::ModeE Mode,
       BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       Benchmark::ResultAggregationModeE ResultAggMode,
-      BenchmarkRunner::ExecutionModeE ExecutionMode) const;
+      BenchmarkRunner::ExecutionModeE ExecutionMode,
+      unsigned BenchmarkRepeatCount) const;
   std::unique_ptr<BenchmarkRunner> virtual createUopsBenchmarkRunner(
       const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector,
       Benchmark::ResultAggregationModeE ResultAggMode,
diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
index 261335a817d06..57330a8ab0fb7 100644
--- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
+++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp
@@ -262,6 +262,12 @@ static cl::opt<BenchmarkRunner::ExecutionModeE> ExecutionMode(
                           "allows for the use of memory annotations")),
     cl::init(BenchmarkRunner::ExecutionModeE::InProcess));
 
+static cl::opt<unsigned>
+    BenchmarkRepeatCount("benchmark-repeat-count",
+                         cl::desc("The number of times to repeat the benchmark "
+                                  "before aggregating the results"),
+                         cl::cat(BenchmarkOptions), cl::init(30));
+
 static ExitOnError ExitOnErr("llvm-exegesis error: ");
 
 // Helper function that logs the error(s) and exits.