Merge pull request #1481 from o1-labs/feature/benchmarks

Benchmark runner

CHANGELOG.md

@@ -17,6 +17,10 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 ## [Unreleased](https://github.com/o1-labs/o1js/compare/74948acac...HEAD)
 
+### Added
+
+- Internal benchmarking tooling to keep track of performance https://github.com/o1-labs/o1js/pull/1481
+
 ## [0.17.0](https://github.com/o1-labs/o1js/compare/1ad7333e9e...74948acac) - 2024-03-06
 
 ### Breaking changes

benchmarks/benchmark.ts

@@ -0,0 +1,158 @@
+/**
+ * Benchmark runner
+ */
+import jStat from 'jstat';
+export { BenchmarkResult, Benchmark, benchmark, printResult, pValue };
+
+type BenchmarkResult = {
+  label: string;
+  size: number;
+  mean: number;
+  variance: number;
+};
+
+type Benchmark = { run: () => Promise<BenchmarkResult[]> };
+
+function benchmark(
+  label: string,
+  run:
+    | ((
+        tic: (label?: string) => void,
+        toc: (label?: string) => void
+      ) => Promise<void>)
+    | ((tic: (label?: string) => void, toc: (label?: string) => void) => void),
+  options?: {
+    numberOfRuns?: number;
+    numberOfWarmups?: number;
+  }
+): Benchmark {
+  return {
+    async run() {
+      const { numberOfRuns = 5, numberOfWarmups = 0 } = options ?? {};
+
+      let lastStartKey: string;
+      let startTime: Record<string, number | undefined> = {}; // key: startTime
+      let runTimes: Record<string, number[]> = {}; // key: [(endTime - startTime)]
+
+      function reset() {
+        startTime = {};
+      }
+
+      function start(key?: string) {
+        lastStartKey = key ?? '';
+        key = getKey(label, key);
+        if (startTime[key] !== undefined)
+          throw Error('running `start(label)` with an already started label');
+        startTime[key] = performance.now();
+      }
+
+      function stop(key?: string) {
+        let end = performance.now();
+        key ??= lastStartKey;
+        if (key === undefined) {
+          throw Error('running `stop()` with no start defined');
+        }
+        key = getKey(label, key);
+        let start_ = startTime[key];
+        startTime[key] = undefined;
+        if (start_ === undefined)
+          throw Error('running `stop()` with no start defined');
+        let times = (runTimes[key] ??= []);
+        times.push(end - start_);
+      }
+
+      let noop = () => {};
+      for (let i = 0; i < numberOfWarmups; i++) {
+        reset();
+        await run(noop, noop);
+      }
+      for (let i = 0; i < numberOfRuns; i++) {
+        reset();
+        await run(start, stop);
+      }
+
+      const results: BenchmarkResult[] = [];
+
+      for (let label in runTimes) {
+        let times = runTimes[label];
+        results.push({ label, ...getStatistics(times) });
+      }
+      return results;
+    },
+  };
+}
+
+function getKey(label: string, key?: string) {
+  return key ? `${label} - ${key}` : label;
+}
+
+function getStatistics(numbers: number[]) {
+  let sum = 0;
+  let sumSquares = 0;
+  for (let i of numbers) {
+    sum += i;
+    sumSquares += i ** 2;
+  }
+  let n = numbers.length;
+  let mean = sum / n;
+  let variance = (sumSquares - sum ** 2 / n) / (n - 1);
+
+  return { mean, variance, size: n };
+}
+
+function printResult(
+  result: BenchmarkResult,
+  previousResult?: BenchmarkResult
+) {
+  console.log(result.label + `\n`);
+  console.log(`time: ${resultToString(result)}`);
+
+  if (previousResult === undefined) return;
+
+  let change = (result.mean - previousResult.mean) / previousResult.mean;
+  let p = pValue(result, previousResult);
+
+  let changePositive = change > 0 ? '+' : '';
+  let pGreater = p > 0.05 ? '>' : '<';
+  console.log(
+    `change: ${changePositive}${(change * 100).toFixed(3)}% (p = ${p.toFixed(
+      2
+    )} ${pGreater} 0.05)`
+  );
+
+  if (p < 0.05) {
+    if (result.mean < previousResult.mean) {
+      console.log('Performance has improved');
+    } else {
+      console.log('Performance has regressed');
+    }
+  } else {
+    console.log('Change within noise threshold.');
+  }
+  console.log('\n');
+}
+
+function resultToString({ mean, variance }: BenchmarkResult) {
+  return `${mean.toFixed(3)}ms ± ${((Math.sqrt(variance) / mean) * 100).toFixed(
+    1
+  )}%`;
+}
+
+function pValue(sample1: BenchmarkResult, sample2: BenchmarkResult): number {
+  const n1 = sample1.size;
+  const n2 = sample2.size;
+  const mean1 = sample1.mean;
+  const mean2 = sample2.mean;
+  const var1 = sample1.variance / n1;
+  const var2 = sample2.variance / n2;
+
+  // calculate the t-statistic
+  const tStatistic = (mean1 - mean2) / Math.sqrt(var1 + var2);
+
+  // degrees of freedom
+  const df = (var1 + var2) ** 2 / (var1 ** 2 / (n1 - 1) + var2 ** 2 / (n2 - 1));
+
+  // calculate the (two-sided) p-value indicating a significant change
+  const pValue = 2 * (1 - jStat.studentt.cdf(Math.abs(tStatistic), df));
+  return pValue;
+}

benchmarks/ecdsa.ts

@@ -0,0 +1,71 @@
+/**
+ * Benchmark runner example
+ *
+ * Run with
+ * ```
+ * ./run benchmarks/ecdsa.ts --bundle
+ * ```
+ */
+import { Provable } from 'o1js';
+import {
+  keccakAndEcdsa,
+  Secp256k1,
+  Ecdsa,
+  Bytes32,
+} from '../src/examples/crypto/ecdsa/ecdsa.js';
+import { BenchmarkResult, benchmark, printResult } from './benchmark.js';
+
+let privateKey = Secp256k1.Scalar.random();
+let publicKey = Secp256k1.generator.scale(privateKey);
+let message = Bytes32.fromString("what's up");
+let signature = Ecdsa.sign(message.toBytes(), privateKey.toBigInt());
+
+const EcdsaBenchmark = benchmark(
+  'ecdsa',
+  async (tic, toc) => {
+    tic('build constraint system');
+    keccakAndEcdsa.analyzeMethods();
+    toc();
+
+    tic('witness generation');
+    Provable.runAndCheck(() => {
+      let message_ = Provable.witness(Bytes32.provable, () => message);
+      let signature_ = Provable.witness(Ecdsa.provable, () => signature);
+      let publicKey_ = Provable.witness(Secp256k1.provable, () => publicKey);
+      keccakAndEcdsa.rawMethods.verifyEcdsa(message_, signature_, publicKey_);
+    });
+    toc();
+  },
+  // two warmups to ensure full caching
+  { numberOfWarmups: 2, numberOfRuns: 5 }
+);
+
+// mock: load previous results
+
+let previousResults: BenchmarkResult[] = [
+  {
+    label: 'ecdsa - build constraint system',
+    mean: 3103.639612600001,
+    variance: 72678.9751211293,
+    size: 5,
+  },
+  {
+    label: 'ecdsa - witness generation',
+    mean: 2062.8708897999995,
+    variance: 13973.913943626918,
+    size: 5,
+  },
+];
+
+// run benchmark
+
+let results = await EcdsaBenchmark.run();
+
+// example for how to log results
+// criterion-style comparison of result to previous one, check significant improvement
+
+for (let i = 0; i < results.length; i++) {
+  let result = results[i];
+  let previous = previousResults[i];
+  printResult(result, previous);
+}

benchmarks/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../tsconfig.json",
+  "include": ["."],
+  "exclude": [],
+  "compilerOptions": {
+    "rootDir": "..",
+    "baseUrl": "..",
+    "paths": {
+      "o1js": ["."]
+    }
+  }
+}

benchmarks/types.d.ts

@@ -0,0 +1 @@
+declare module 'jstat';

package.json

@@ -82,6 +82,7 @@
     "glob": "^8.0.3",
     "howslow": "^0.1.0",
     "jest": "^28.1.3",
+    "jstat": "^1.9.6",
     "minimist": "^1.2.7",
     "prettier": "^2.8.4",
     "replace-in-file": "^6.3.5",