feat: add benchmark relative speed ratio error margin to summary

Signed-off-by: Jérôme Benoit <jerome.benoit@piment-noir.org>

CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to
 
 ## [Unreleased]
 
+###
+
+- Add benchmark relative speed ratio error margin to summary.
+
 ## [0.4.10] - 2024-05-16
 
 ### Fixed

src/reporter/fmt.js

@@ -41,8 +41,8 @@ export function errorMargin(rmoe) {
     .replaceAll(',', "'")} %`;
 }
 
-export function speedRatio(speed) {
-  return `${Number(speed.toFixed(2))
+export function speedRatio(ratio) {
+  return `${Number(ratio.toFixed(2))
     .toLocaleString(locale)
     .replaceAll(',', "'")}`;
 }

src/reporter/table.js

@@ -1,4 +1,4 @@
-import { tatamiNgGroup } from '../constants.js';
+import { tTable, tatamiNgGroup } from '../constants.js';
 import * as clr from './clr.js';
 import { duration, errorMargin, iterPerSecond, speedRatio } from './fmt.js';
 
@@ -140,13 +140,27 @@ export function summary(benchmarks, { colors = true }) {
     .filter(benchmark => benchmark !== baseline)
     .map(benchmark => {
       const ratio = benchmark.stats.avg / baseline.stats.avg;
+      // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae
+      const ratioSd =
+        ratio *
+        Math.sqrt(
+          (baseline.stats.sd / baseline.stats.avg) ** 2 +
+            (benchmark.stats.sd / benchmark.stats.avg) ** 2,
+        );
+      const ratioSem =
+        ratioSd / Math.sqrt(baseline.stats.samples + benchmark.stats.samples);
+      const critical =
+        tTable[
+          (baseline.stats.samples + benchmark.stats.samples - 1 || 1).toString()
+        ] || tTable.infinity;
+      const ratioMoe = ratioSem * critical;
+      const ratioRmoe = (ratioMoe / ratio) * 100;
       return `\n   ${clr[1 > ratio ? 'red' : 'green'](
         colors,
         1 > ratio ? speedRatio(1 / ratio) : speedRatio(ratio),
-      )}x ${1 > ratio ? 'slower' : 'faster'} than ${clr.bold(
-        colors,
-        clr.cyan(colors, benchmark.name),
-      )}`;
+      )} ± ${clr.yellow(colors, errorMargin(ratioRmoe))} times ${
+        1 > ratio ? 'slower' : 'faster'
+      } than ${clr.bold(colors, clr.cyan(colors, benchmark.name))}`;
     })
     .join('')}`;
 }