feat(examples-plugins): add example plugin benchmark-js

code-pushup.config.ts

@@ -1,8 +1,13 @@
 import 'dotenv/config';
+import { join } from 'node:path';
 import { z } from 'zod';
 import {
+  JS_BENCHMARKING_PLUGIN_SLUG,
+  JS_BENCHMARKING_TINYBENCH_RUNNER_PATH,
   fileSizePlugin,
   fileSizeRecommendedRefs,
+  jsBenchmarkingPlugin,
+  jsBenchmarkingSuiteNameToCategoryRef,
   packageJsonDocumentationGroupRef,
   packageJsonPerformanceGroupRef,
   packageJsonPlugin,
@@ -31,6 +36,8 @@ const envSchema = z
   .partial();
 const env = await envSchema.parseAsync(process.env);
 
+const benchmarkJsSuiteNames = ['score-report'];
+
 const config: CoreConfig = {
   ...(env.CP_SERVER &&
     env.CP_API_KEY &&
@@ -78,6 +85,15 @@ const config: CoreConfig = {
     }),
 
     await lighthousePlugin('https://codepushup.dev/'),
+
+    await jsBenchmarkingPlugin({
+      tsconfig: join('packages', 'utils', 'tsconfig.perf.ts'),
+      runnerPath: JS_BENCHMARKING_TINYBENCH_RUNNER_PATH,
+      outputDir: join('.code-pushup', JS_BENCHMARKING_PLUGIN_SLUG),
+      targets: benchmarkJsSuiteNames.map(suit =>
+        join('packages', 'utils', 'perf', suit, 'index.ts'),
+      ),
+    }),
   ],
 
   categories: [
@@ -168,6 +184,7 @@ const config: CoreConfig = {
         ...fileSizeRecommendedRefs,
         packageJsonPerformanceGroupRef,
         packageJsonDocumentationGroupRef,
+        ...benchmarkJsSuiteNames.map(jsBenchmarkingSuiteNameToCategoryRef),
       ],
     },
   ],

examples/plugins/code-pushup.config.ts

@@ -3,6 +3,8 @@ import {
   LIGHTHOUSE_OUTPUT_FILE_DEFAULT,
   fileSizePlugin,
   fileSizeRecommendedRefs,
+  jsBenchmarkingPlugin,
+  jsBenchmarkingSuiteNameToCategoryRef,
   lighthouseCorePerfGroupRefs,
   lighthousePlugin,
   packageJsonDocumentationGroupRef,
@@ -20,6 +22,9 @@ import {
  *
  */
 
+const projectRoot = join('examples', 'plugins');
+const benchmarkJsSuiteNames = ['dummy-suite'];
+
 const config = {
   plugins: [
     fileSizePlugin({
@@ -42,6 +47,12 @@ const config = {
       headless: false,
       verbose: true,
     }),
+    await jsBenchmarkingPlugin({
+      targets: benchmarkJsSuiteNames.map(folder =>
+        join(projectRoot, 'perf', folder, 'index.ts'),
+      ),
+      tsconfig: join(projectRoot, 'tsconfig.perf.json'),
+    }),
   ],
   categories: [
     {
@@ -51,6 +62,7 @@ const config = {
         ...fileSizeRecommendedRefs,
         packageJsonPerformanceGroupRef,
         ...lighthouseCorePerfGroupRefs,
+        ...benchmarkJsSuiteNames.map(jsBenchmarkingSuiteNameToCategoryRef),
       ],
     },
     {

examples/plugins/perf/dummy-suite/factorial.ts

@@ -0,0 +1,3 @@
+export function factorial(n: number): number {
+  return n === 0 || n === 1 ? 1 : n * factorial(n - 1);
+}

examples/plugins/perf/dummy-suite/index.ts

@@ -0,0 +1,71 @@
+import yargs from 'yargs';
+import type { SuiteConfig } from '../../src/js-benchmarking/src/runner/types';
+import { factorial } from './factorial';
+
+const cli = yargs(process.argv).options({
+  numCases: {
+    type: 'number',
+    default: 2,
+  },
+  executionTime: {
+    type: 'number',
+    default: 0,
+  },
+  executionTimeDiff: {
+    type: 'number',
+    default: 0,
+  },
+  syncIterations: {
+    type: 'number',
+    default: 500,
+  },
+  syncIterationsDiff: {
+    type: 'number',
+    default: 1000,
+  },
+  logs: {
+    type: 'boolean',
+    default: false,
+  },
+});
+
+const {
+  numCases,
+  executionTime,
+  executionTimeDiff,
+  syncIterations,
+  syncIterationsDiff,
+  logs, // eslint-disable-next-line n/no-sync
+} = cli.parseSync();
+
+if (logs) {
+  // eslint-disable-next-line no-console
+  console.log('You can adjust the test with the following arguments:');
+  // eslint-disable-next-line no-console
+  console.log(
+    `numCases           number of test cases                    --numCases=${numCases.toString()}`,
+    `executionTime      duration of first case in ms            --executionTime=${executionTime.toString()}`,
+    `executionTimeDiff  time diff in execution duration in ms   --executionTimeDiff=${executionTimeDiff.toString()}`,
+    `syncIterations     executions of dummy fn factorial        --syncIterations=${syncIterations.toString()}`,
+    `syncIterationsDiff diff in number of executions            --syncIterationsDiff=${syncIterationsDiff.toString()}`,
+  );
+}
+
+// ==================
+
+const suiteConfig: SuiteConfig = {
+  suiteName: 'dummy-suite',
+  targetImplementation: 'case-1',
+  cases: Array.from({ length: numCases }).map((_, idx) => [
+    `case-${idx + 1}`,
+    () =>
+      new Promise(resolve =>
+        setTimeout(() => {
+          resolve(factorial((syncIterations + syncIterationsDiff) * idx));
+        }, executionTime + executionTimeDiff * idx),
+      ),
+  ]),
+  time: executionTime + executionTimeDiff * 2,
+};
+
+export default suiteConfig;

examples/plugins/project.json

@@ -12,7 +12,12 @@
         "main": "examples/plugins/src/index.ts",
         "tsConfig": "examples/plugins/tsconfig.lib.json",
         "assets": ["examples/plugins/*.md"],
-        "esbuildConfig": "esbuild.config.js"
+        "esbuildConfig": "esbuild.config.js",
+        "additionalEntryPoints": [
+          "examples/plugins/src/js-benchmarking.benchmark.runner.ts",
+          "examples/plugins/src/js-benchmarking.benny.runner.ts",
+          "examples/plugins/src/js-benchmarking.tinybench.runner.ts"
+        ]
       }
     },
     "lint": {
@@ -70,6 +75,9 @@
           "target": "build"
         }
       ]
+    },
+    "perf": {
+      "command": "node tools/benchmark/bin.mjs --targets ./examples/plugins/perf/dummy-suite/index.ts"
     }
   },
   "tags": ["scope:internal", "type:feature"]

examples/plugins/src/index.ts

@@ -16,3 +16,13 @@ export {
   LIGHTHOUSE_OUTPUT_FILE_DEFAULT,
   recommendedRefs as lighthouseCorePerfGroupRefs,
 } from './lighthouse/src/index';
+export {
+  jsBenchmarkingPlugin,
+  jsBenchmarkingSuiteNameToCategoryRef,
+  JsBenchmarkingPluginConfig,
+  JS_BENCHMARKING_PLUGIN_SLUG,
+  JS_BENCHMARKING_BENCHMARK_RUNNER_PATH,
+  JS_BENCHMARKING_BENNY_RUNNER_PATH,
+  JS_BENCHMARKING_TINYBENCH_RUNNER_PATH,
+  JS_BENCHMARKING_DEFAULT_RUNNER_PATH,
+} from './js-benchmarking/src/index';

examples/plugins/src/js-benchmarking.benchmark.runner.ts

@@ -0,0 +1,3 @@
+import { benchmarkRunner } from './js-benchmarking/src/runner/benchmark.suite-runner';
+
+export default benchmarkRunner;

examples/plugins/src/js-benchmarking.benny.runner.ts

@@ -0,0 +1,3 @@
+import { bennyRunner } from './js-benchmarking/src/runner/benny.suite-runner';
+
+export default bennyRunner;

examples/plugins/src/js-benchmarking.tinybench.runner.ts

@@ -0,0 +1,3 @@
+import { tinybenchRunner } from './js-benchmarking/src/runner/tinybench.suite-runner';
+
+export default tinybenchRunner;

examples/plugins/src/js-benchmarking/README.md

@@ -0,0 +1,132 @@
+# benchmark js example
+
+🕵️️ **Code PushUp plugin to benchmark JS execution performance** 📊
+
+---
+
+The plugin analyzes a given suite name and creates benchmark audits.  
+It uses [tinybench](https://github.com/tinylibs/tinybench) under the hood.
+
+You can configure the plugin with the following options:
+
+- `targets` - files to load that export a suite
+- `tsconfig` - path to tsconfig file _(optional)_
+- `logs` - additional information _(optional)_
+
+## Getting started
+
+1. If you haven't already, install [@code-pushup/cli](../../../../packages/cli/README.md) and create a configuration file.
+
+2. Copy the [plugin source](./src/) as is into your project
+
+3. Add this plugin to the `plugins` array in your Code PushUp CLI config file (e.g. `code-pushup.config.js`).
+
+   Pass in the path on the directory to load the test suite form (relative to `process.cwd()`), for more options see [BenchmarkJsPluginOptions]().
+
+   ```js
+   import { join } from 'node:path';
+   import benchmarkJsPlugin from './benchmark-js.plugin';
+
+   export default {
+     // ...
+     plugins: [
+       // ...
+       await benchmarkJsPlugin({
+         targets: ['suites/score-report.ts'],
+       }),
+     ],
+   };
+   ```
+
+   1. Create benchmark suite:
+
+      ```ts
+      // typescript
+      const suiteConfig = {
+        suiteName: 'glob',
+        targetImplementation: 'version-2',
+        cases: [
+          ['version-1', () => new Promise(resolve => setTimeout(resolve, 30))],
+          ['version-2', () => new Promise(resolve => setTimeout(resolve, 10))],
+          ['version-3', () => new Promise(resolve => setTimeout(resolve, 20))],
+        ],
+      };
+      ```
+
+4. (Optional) Set up categories (use `npx code-pushup print-config` to list audits and groups).
+
+   ```js
+   import benchmarkJsPlugin, { suitesToCategorieGroupRef } from './benchmark-js.plugin';
+
+   export default {
+     // ...
+     categories: [
+       // ...
+       {
+         slug: 'performance',
+         title: 'Performance',
+         refs: suitesToCategorieGroupRef(suites),
+       },
+     ],
+   };
+   ```
+
+5. Run the CLI with `npx code-pushup collect` and view or upload report (refer to [CLI docs](../../../../packages/cli/README.m)).
+
+For a standalone usage uf the test runner use our helpers
+
+## Audits
+
+The plugin creates an audit for each suite.
+
+The audit scoring is based on fastest case, that means the fastest audit has a score of 100.
+If the target implementation is not the fastest, the audit shows how much slower the target implementation is compared to the fastest.
+
+`● crawl-file-system - Benchmark JS           59.9 ops/sec`
+
+### Issues
+
+Each audit has the test cases listed as issue.
+
+**Possible issues:**
+
+- is slower - `version-1    59.90 ops/sec (20% slower)`
+- is target and slower - `🎯 version-1    59.90 ops/sec (20% slower)`
+- is fastest - `version-1     59.90 ops/sec 🔥 `
+- is target and fastest - `🎯 version-1     59.90 ops/sec 🔥`
+
+<img width="600px" src="./docs/images/audits-readme-example.png">
+
+## Standalone helper
+
+The plugin also provides a helper function to execute a test suite.
+
+```ts
+import { SuiteConfig, runSuit } from './suite-helper.ts';
+
+const suite: SuiteConfig = {
+  suiteName: 'dummy-suite',
+  targetImplementation: 'version-2',
+  cases: [
+    ['version-1', async () => new Promise(resolve => setTimeout(resolve, 30))],
+    ['version-2', async () => new Promise(resolve => setTimeout(resolve, 50))],
+    ['version-3', async () => new Promise(resolve => setTimeout(resolve, 80))],
+  ],
+};
+const results = await runSuite(suite);
+
+const { suiteName, name, hz: maxHz } = results.find(({ isFastest }) => isFastest);
+const target = results.find(({ isTarget }) => isTarget);
+console.log(`In suite ${suiteName} fastest is: ${name} target is ${target?.name}`);
+console.table(
+  results.map(({ name, hz, rme, samples, isTarget, isFastest }) => {
+    const targetIcon = isTarget ? '🎯' : '';
+    const postfix = isFastest ? '(fastest 🔥)' : `(${((1 - hz / maxHz) * 100).toFixed(1)}% slower)`;
+    return {
+      // fast-glob x 40,824 ops/sec ±4.44% (85 runs sampled)
+      message: `${targetIcon}${name} x ${hz.toFixed(2)} ops/sec ±${rme.toFixed(2)}; ${samples} samples ${postfix}`,
+      severity: hz < maxHz && isTarget ? 'error' : 'info',
+    };
+  }),
+);
+```

examples/plugins/src/js-benchmarking/src/config.ts

@@ -0,0 +1,22 @@
+import { z } from 'zod';
+import { JS_BENCHMARKING_DEFAULT_RUNNER_PATH } from './constants';
+
+export const jsBenchmarkingRunnerOptionsSchema = z.object({
+  runnerPath: z.string().default(JS_BENCHMARKING_DEFAULT_RUNNER_PATH),
+  tsconfig: z.string().optional(),
+  outputDir: z.string().optional(),
+  outputFileName: z.string().optional(),
+  verbose: z.boolean().optional(),
+});
+
+export const jsBenchmarkingPluginOptionsSchema = z.object({
+  targets: z.array(z.string()),
+  runnerPath: z.string().default(JS_BENCHMARKING_DEFAULT_RUNNER_PATH),
+  tsconfig: z.string().optional(),
+  outputDir: z.string().optional(),
+  verbose: z.boolean().optional(),
+});
+
+export type JsBenchmarkingPluginConfig = z.input<
+  typeof jsBenchmarkingPluginOptionsSchema
+>;

examples/plugins/src/js-benchmarking/src/constants.ts

@@ -0,0 +1,21 @@
+import { join } from 'node:path';
+
+export const JS_BENCHMARKING_PLUGIN_SLUG = 'js-benchmarking';
+
+function withRunnerRoot(runnerName: string): string {
+  // @TODO replace with `@code-pushup/js-benchmarking-plugin/src/runner/`
+  return join(
+    'dist',
+    'examples',
+    'plugins',
+    `${JS_BENCHMARKING_PLUGIN_SLUG}.${runnerName}.runner.js`,
+  );
+}
+
+export const JS_BENCHMARKING_TINYBENCH_RUNNER_PATH =
+  withRunnerRoot('tinybench');
+export const JS_BENCHMARKING_BENCHMARK_RUNNER_PATH =
+  withRunnerRoot('benchmark');
+export const JS_BENCHMARKING_BENNY_RUNNER_PATH = withRunnerRoot('benny');
+export const JS_BENCHMARKING_DEFAULT_RUNNER_PATH =
+  JS_BENCHMARKING_TINYBENCH_RUNNER_PATH;