fix: Catch2 v.3.5.0 changed output format [#241]

* add test case for catch2 v3 * modify extract header regexp to work with v3
benchmark-action · May 19, 2024 · e32befd · e32befd
1 parent b481d47
commit e32befd
Show file tree

Hide file tree

Showing 5 changed files with 103 additions and 3 deletions.
diff --git a/src/extract.ts b/src/extract.ts
@@ -473,7 +473,7 @@ function extractCatch2Result(output: string): BenchmarkResult[] {
     //                43.186 us     41.402 us     46.246 us <-- Actual benchmark data
     //                11.719 us      7.847 us     17.747 us <-- Ignored
 
-    const reTestCaseStart = /^benchmark name +samples +iterations +estimated/;
+    const reTestCaseStart = /^benchmark name +samples +iterations +(estimated|est run time)/;
     const reBenchmarkStart = /(\d+) +(\d+) +(?:\d+(\.\d+)?) (?:ns|ms|us|s)\s*$/;
     const reBenchmarkValues =
         /^ +(\d+(?:\.\d+)?) (ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s) +(?:\d+(?:\.\d+)?) (?:ns|us|ms|s)/;

diff --git a/test/__snapshots__/extract.spec.ts.snap b/test/__snapshots__/extract.spec.ts.snap
@@ -245,7 +245,56 @@ exports[`extractResult() extracts benchmark output from cargo - criterion_output
 }
 `;
 
-exports[`extractResult() extracts benchmark output from catch2 - catch2_output.txt 1`] = `
+exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v2.txt 1`] = `
+{
+  "benches": [
+    {
+      "extra": "100 samples
+208 iterations",
+      "name": "Fibonacci 10",
+      "range": "± 19",
+      "unit": "ns",
+      "value": 344,
+    },
+    {
+      "extra": "100 samples
+2 iterations",
+      "name": "Fibonacci 20",
+      "range": "± 3.256",
+      "unit": "us",
+      "value": 41.731,
+    },
+    {
+      "extra": "100 samples
+1961 iterations",
+      "name": "Fibonacci~ 5!",
+      "range": "± 4",
+      "unit": "ns",
+      "value": 36,
+    },
+    {
+      "extra": "100 samples
+20 iterations",
+      "name": "Fibonacci-15_bench",
+      "range": "± 362",
+      "unit": "us",
+      "value": 3.789,
+    },
+  ],
+  "commit": {
+    "author": null,
+    "committer": null,
+    "id": "123456789abcdef",
+    "message": "this is dummy",
+    "timestamp": "dummy timestamp",
+    "url": "https://github.com/dummy/repo",
+  },
+  "date": 1712131503296,
+  "tool": "catch2",
+}
+`;
+
+exports[`extractResult() extracts benchmark output from catch2 - catch2_output_v3.txt 1`] = `
 {
   "benches": [
     {

diff --git a/test/data/extract/catch2_output.txt → test/data/extract/catch2_output_v2.txt b/test/data/extract/catch2_output.txt → test/data/extract/catch2_output_v2.txt
diff --git a/test/data/extract/catch2_output_v3.txt b/test/data/extract/catch2_output_v3.txt
@@ -0,0 +1,47 @@
+
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Catch2_bench is a Catch v2.11.0 host application.
+Run with -? for options
+
+-------------------------------------------------------------------------------
+Fibonacci
+-------------------------------------------------------------------------------
+/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:5
+...............................................................................
+
+benchmark name                                  samples       iterations    est run time
+                                                mean          low mean      high mean
+                                                std dev       low std dev   high std dev
+-------------------------------------------------------------------------------
+Fibonacci 10                                              100           208     7.1968 ms 
+                                                       344 ns        341 ns        349 ns 
+                                                        19 ns         11 ns         29 ns 
+
+Fibonacci 20                                              100             2     8.3712 ms 
+                                                    41.731 us      41.25 us     42.622 us 
+                                                     3.256 us      2.163 us      5.353 us 
+
+
+-------------------------------------------------------------------------------
+More Fibonacci
+-------------------------------------------------------------------------------
+/Users/rhayasd/Develop/github.com/benchmark-action/github-action-benchmark/examples/catch2/catch2_bench.cpp:13
+...............................................................................
+
+benchmark name                                  samples       iterations    estimated
+                                                mean          low mean      high mean
+                                                std dev       low std dev   high std dev
+-------------------------------------------------------------------------------
+Fibonacci~ 5!                                             100          1961     7.0596 ms 
+                                                        36 ns         35 ns         37 ns 
+                                                         4 ns          3 ns          6 ns 
+
+Fibonacci-15_bench                                        100            20       7.48 ms 
+                                                     3.789 us      3.734 us      3.888 us 
+                                                       362 ns        234 ns        539 ns 
+
+
+===============================================================================
+test cases: 2 | 2 passed
+assertions: - none -
+
diff --git a/test/extract.spec.ts b/test/extract.spec.ts
@@ -77,7 +77,11 @@ describe('extractResult()', function () {
         },
         {
             tool: 'catch2',
-            file: 'catch2_output.txt',
+            file: 'catch2_output_v2.txt',
+        },
+        {
+            tool: 'catch2',
+            file: 'catch2_output_v3.txt',
         },
         {
             tool: 'go',