diff --git a/code-pushup.config.ts b/code-pushup.config.ts index b917977f3..b0bdd0c31 100644 --- a/code-pushup.config.ts +++ b/code-pushup.config.ts @@ -1,8 +1,13 @@ import 'dotenv/config'; +import { join } from 'node:path'; import { z } from 'zod'; import { + JS_BENCHMARKING_PLUGIN_SLUG, + JS_BENCHMARKING_TINYBENCH_RUNNER_PATH, fileSizePlugin, fileSizeRecommendedRefs, + jsBenchmarkingPlugin, + jsBenchmarkingSuiteNameToCategoryRef, packageJsonDocumentationGroupRef, packageJsonPerformanceGroupRef, packageJsonPlugin, @@ -31,6 +36,8 @@ const envSchema = z .partial(); const env = await envSchema.parseAsync(process.env); +const benchmarkJsSuiteNames = ['score-report']; + const config: CoreConfig = { ...(env.CP_SERVER && env.CP_API_KEY && @@ -78,6 +85,15 @@ const config: CoreConfig = { }), await lighthousePlugin('https://codepushup.dev/'), + + await jsBenchmarkingPlugin({ + tsconfig: join('packages', 'utils', 'tsconfig.perf.ts'), + runnerPath: JS_BENCHMARKING_TINYBENCH_RUNNER_PATH, + outputDir: join('.code-pushup', JS_BENCHMARKING_PLUGIN_SLUG), + targets: benchmarkJsSuiteNames.map(suit => + join('packages', 'utils', 'perf', suit, 'index.ts'), + ), + }), ], categories: [ @@ -168,6 +184,7 @@ const config: CoreConfig = { ...fileSizeRecommendedRefs, packageJsonPerformanceGroupRef, packageJsonDocumentationGroupRef, + ...benchmarkJsSuiteNames.map(jsBenchmarkingSuiteNameToCategoryRef), ], }, ], diff --git a/examples/plugins/code-pushup.config.ts b/examples/plugins/code-pushup.config.ts index ad61ae851..74cf2f97b 100644 --- a/examples/plugins/code-pushup.config.ts +++ b/examples/plugins/code-pushup.config.ts @@ -3,6 +3,8 @@ import { LIGHTHOUSE_OUTPUT_FILE_DEFAULT, fileSizePlugin, fileSizeRecommendedRefs, + jsBenchmarkingPlugin, + jsBenchmarkingSuiteNameToCategoryRef, lighthouseCorePerfGroupRefs, lighthousePlugin, packageJsonDocumentationGroupRef, @@ -20,6 +22,9 @@ import { * */ +const projectRoot = join('examples', 'plugins'); +const benchmarkJsSuiteNames = ['dummy-suite']; + const config = { plugins: [ fileSizePlugin({ @@ -42,6 +47,12 @@ const config = { headless: false, verbose: true, }), + await jsBenchmarkingPlugin({ + targets: benchmarkJsSuiteNames.map(folder => + join(projectRoot, 'perf', folder, 'index.ts'), + ), + tsconfig: join(projectRoot, 'tsconfig.perf.json'), + }), ], categories: [ { @@ -51,6 +62,7 @@ const config = { ...fileSizeRecommendedRefs, packageJsonPerformanceGroupRef, ...lighthouseCorePerfGroupRefs, + ...benchmarkJsSuiteNames.map(jsBenchmarkingSuiteNameToCategoryRef), ], }, { diff --git a/examples/plugins/perf/dummy-suite/factorial.ts b/examples/plugins/perf/dummy-suite/factorial.ts new file mode 100644 index 000000000..103c6875d --- /dev/null +++ b/examples/plugins/perf/dummy-suite/factorial.ts @@ -0,0 +1,3 @@ +export function factorial(n: number): number { + return n === 0 || n === 1 ? 1 : n * factorial(n - 1); +} diff --git a/examples/plugins/perf/dummy-suite/index.ts b/examples/plugins/perf/dummy-suite/index.ts new file mode 100644 index 000000000..66310597f --- /dev/null +++ b/examples/plugins/perf/dummy-suite/index.ts @@ -0,0 +1,71 @@ +import yargs from 'yargs'; +import type { SuiteConfig } from '../../src/js-benchmarking/src/runner/types'; +import { factorial } from './factorial'; + +const cli = yargs(process.argv).options({ + numCases: { + type: 'number', + default: 2, + }, + executionTime: { + type: 'number', + default: 0, + }, + executionTimeDiff: { + type: 'number', + default: 0, + }, + syncIterations: { + type: 'number', + default: 500, + }, + syncIterationsDiff: { + type: 'number', + default: 1000, + }, + logs: { + type: 'boolean', + default: false, + }, +}); + +const { + numCases, + executionTime, + executionTimeDiff, + syncIterations, + syncIterationsDiff, + logs, // eslint-disable-next-line n/no-sync +} = cli.parseSync(); + +if (logs) { + // eslint-disable-next-line no-console + console.log('You can adjust the test with the following arguments:'); + // eslint-disable-next-line no-console + console.log( + `numCases number of test cases --numCases=${numCases.toString()}`, + `executionTime duration of first case in ms --executionTime=${executionTime.toString()}`, + `executionTimeDiff time diff in execution duration in ms --executionTimeDiff=${executionTimeDiff.toString()}`, + `syncIterations executions of dummy fn factorial --syncIterations=${syncIterations.toString()}`, + `syncIterationsDiff diff in number of executions --syncIterationsDiff=${syncIterationsDiff.toString()}`, + ); +} + +// ================== + +const suiteConfig: SuiteConfig = { + suiteName: 'dummy-suite', + targetImplementation: 'case-1', + cases: Array.from({ length: numCases }).map((_, idx) => [ + `case-${idx + 1}`, + () => + new Promise(resolve => + setTimeout(() => { + resolve(factorial((syncIterations + syncIterationsDiff) * idx)); + }, executionTime + executionTimeDiff * idx), + ), + ]), + time: executionTime + executionTimeDiff * 2, +}; + +export default suiteConfig; diff --git a/examples/plugins/project.json b/examples/plugins/project.json index f5eb63b79..82b251a65 100644 --- a/examples/plugins/project.json +++ b/examples/plugins/project.json @@ -12,7 +12,12 @@ "main": "examples/plugins/src/index.ts", "tsConfig": "examples/plugins/tsconfig.lib.json", "assets": ["examples/plugins/*.md"], - "esbuildConfig": "esbuild.config.js" + "esbuildConfig": "esbuild.config.js", + "additionalEntryPoints": [ + "examples/plugins/src/js-benchmarking.benchmark.runner.ts", + "examples/plugins/src/js-benchmarking.benny.runner.ts", + "examples/plugins/src/js-benchmarking.tinybench.runner.ts" + ] } }, "lint": { @@ -70,6 +75,9 @@ "target": "build" } ] + }, + "perf": { + "command": "node tools/benchmark/bin.mjs --targets ./examples/plugins/perf/dummy-suite/index.ts" } }, "tags": ["scope:internal", "type:feature"] diff --git a/examples/plugins/src/index.ts b/examples/plugins/src/index.ts index 4935449df..87a9aa068 100644 --- a/examples/plugins/src/index.ts +++ b/examples/plugins/src/index.ts @@ -16,3 +16,13 @@ export { LIGHTHOUSE_OUTPUT_FILE_DEFAULT, recommendedRefs as lighthouseCorePerfGroupRefs, } from './lighthouse/src/index'; +export { + jsBenchmarkingPlugin, + jsBenchmarkingSuiteNameToCategoryRef, + JsBenchmarkingPluginConfig, + JS_BENCHMARKING_PLUGIN_SLUG, + JS_BENCHMARKING_BENCHMARK_RUNNER_PATH, + JS_BENCHMARKING_BENNY_RUNNER_PATH, + JS_BENCHMARKING_TINYBENCH_RUNNER_PATH, + JS_BENCHMARKING_DEFAULT_RUNNER_PATH, +} from './js-benchmarking/src/index'; diff --git a/examples/plugins/src/js-benchmarking.benchmark.runner.ts b/examples/plugins/src/js-benchmarking.benchmark.runner.ts new file mode 100644 index 000000000..74650566c --- /dev/null +++ b/examples/plugins/src/js-benchmarking.benchmark.runner.ts @@ -0,0 +1,3 @@ +import { benchmarkRunner } from './js-benchmarking/src/runner/benchmark.suite-runner'; + +export default benchmarkRunner; diff --git a/examples/plugins/src/js-benchmarking.benny.runner.ts b/examples/plugins/src/js-benchmarking.benny.runner.ts new file mode 100644 index 000000000..91437cb13 --- /dev/null +++ b/examples/plugins/src/js-benchmarking.benny.runner.ts @@ -0,0 +1,3 @@ +import { bennyRunner } from './js-benchmarking/src/runner/benny.suite-runner'; + +export default bennyRunner; diff --git a/examples/plugins/src/js-benchmarking.tinybench.runner.ts b/examples/plugins/src/js-benchmarking.tinybench.runner.ts new file mode 100644 index 000000000..fc7eca944 --- /dev/null +++ b/examples/plugins/src/js-benchmarking.tinybench.runner.ts @@ -0,0 +1,3 @@ +import { tinybenchRunner } from './js-benchmarking/src/runner/tinybench.suite-runner'; + +export default tinybenchRunner; diff --git a/examples/plugins/src/js-benchmarking/README.md b/examples/plugins/src/js-benchmarking/README.md new file mode 100644 index 000000000..2e23a43a3 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/README.md @@ -0,0 +1,132 @@ +# benchmark js example + +🕵️️ **Code PushUp plugin to benchmark JS execution performance** 📊 + +--- + +The plugin analyzes a given suite name and creates benchmark audits. +It uses [tinybench](https://github.com/tinylibs/tinybench) under the hood. + +You can configure the plugin with the following options: + +- `targets` - files to load that export a suite +- `tsconfig` - path to tsconfig file _(optional)_ +- `logs` - additional information _(optional)_ + +## Getting started + +1. If you haven't already, install [@code-pushup/cli](../../../../packages/cli/README.md) and create a configuration file. + +2. Copy the [plugin source](./src/) as is into your project + +3. Add this plugin to the `plugins` array in your Code PushUp CLI config file (e.g. `code-pushup.config.js`). + + Pass in the path on the directory to load the test suite form (relative to `process.cwd()`), for more options see [BenchmarkJsPluginOptions](). + + ```js + import { join } from 'node:path'; + import benchmarkJsPlugin from './benchmark-js.plugin'; + + export default { + // ... + plugins: [ + // ... + await benchmarkJsPlugin({ + targets: ['suites/score-report.ts'], + }), + ], + }; + ``` + + 1. Create benchmark suite: + + ```ts + // typescript + const suiteConfig = { + suiteName: 'glob', + targetImplementation: 'version-2', + cases: [ + ['version-1', () => new Promise(resolve => setTimeout(resolve, 30))], + ['version-2', () => new Promise(resolve => setTimeout(resolve, 10))], + ['version-3', () => new Promise(resolve => setTimeout(resolve, 20))], + ], + }; + ``` + +4. (Optional) Set up categories (use `npx code-pushup print-config` to list audits and groups). + + ```js + import benchmarkJsPlugin, { suitesToCategorieGroupRef } from './benchmark-js.plugin'; + + export default { + // ... + categories: [ + // ... + { + slug: 'performance', + title: 'Performance', + refs: suitesToCategorieGroupRef(suites), + }, + ], + }; + ``` + +5. Run the CLI with `npx code-pushup collect` and view or upload report (refer to [CLI docs](../../../../packages/cli/README.m)). + +For a standalone usage uf the test runner use our helpers + +## Audits + +The plugin creates an audit for each suite. + +The audit scoring is based on fastest case, that means the fastest audit has a score of 100. +If the target implementation is not the fastest, the audit shows how much slower the target implementation is compared to the fastest. + +`● crawl-file-system - Benchmark JS 59.9 ops/sec` + +### Issues + +Each audit has the test cases listed as issue. + +**Possible issues:** + +- is slower - `version-1 59.90 ops/sec (20% slower)` +- is target and slower - `🎯 version-1 59.90 ops/sec (20% slower)` +- is fastest - `version-1 59.90 ops/sec 🔥 ` +- is target and fastest - `🎯 version-1 59.90 ops/sec 🔥` + + + +## Standalone helper + +The plugin also provides a helper function to execute a test suite. + +```ts +import { SuiteConfig, runSuit } from './suite-helper.ts'; + +const suite: SuiteConfig = { + suiteName: 'dummy-suite', + targetImplementation: 'version-2', + cases: [ + ['version-1', async () => new Promise(resolve => setTimeout(resolve, 30))], + ['version-2', async () => new Promise(resolve => setTimeout(resolve, 50))], + ['version-3', async () => new Promise(resolve => setTimeout(resolve, 80))], + ], +}; +const results = await runSuite(suite); + +const { suiteName, name, hz: maxHz } = results.find(({ isFastest }) => isFastest); +const target = results.find(({ isTarget }) => isTarget); +console.log(`In suite ${suiteName} fastest is: ${name} target is ${target?.name}`); +console.table( + results.map(({ name, hz, rme, samples, isTarget, isFastest }) => { + const targetIcon = isTarget ? '🎯' : ''; + const postfix = isFastest ? '(fastest 🔥)' : `(${((1 - hz / maxHz) * 100).toFixed(1)}% slower)`; + return { + // fast-glob x 40,824 ops/sec ±4.44% (85 runs sampled) + message: `${targetIcon}${name} x ${hz.toFixed(2)} ops/sec ±${rme.toFixed(2)}; ${samples} samples ${postfix}`, + severity: hz < maxHz && isTarget ? 'error' : 'info', + }; + }), +); +``` diff --git a/examples/plugins/src/js-benchmarking/docs/images/audits-readme-example.png b/examples/plugins/src/js-benchmarking/docs/images/audits-readme-example.png new file mode 100644 index 000000000..19687949b Binary files /dev/null and b/examples/plugins/src/js-benchmarking/docs/images/audits-readme-example.png differ diff --git a/examples/plugins/src/js-benchmarking/src/config.ts b/examples/plugins/src/js-benchmarking/src/config.ts new file mode 100644 index 000000000..084715d49 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/config.ts @@ -0,0 +1,22 @@ +import { z } from 'zod'; +import { JS_BENCHMARKING_DEFAULT_RUNNER_PATH } from './constants'; + +export const jsBenchmarkingRunnerOptionsSchema = z.object({ + runnerPath: z.string().default(JS_BENCHMARKING_DEFAULT_RUNNER_PATH), + tsconfig: z.string().optional(), + outputDir: z.string().optional(), + outputFileName: z.string().optional(), + verbose: z.boolean().optional(), +}); + +export const jsBenchmarkingPluginOptionsSchema = z.object({ + targets: z.array(z.string()), + runnerPath: z.string().default(JS_BENCHMARKING_DEFAULT_RUNNER_PATH), + tsconfig: z.string().optional(), + outputDir: z.string().optional(), + verbose: z.boolean().optional(), +}); + +export type JsBenchmarkingPluginConfig = z.input< + typeof jsBenchmarkingPluginOptionsSchema +>; diff --git a/examples/plugins/src/js-benchmarking/src/constants.ts b/examples/plugins/src/js-benchmarking/src/constants.ts new file mode 100644 index 000000000..1750fb148 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/constants.ts @@ -0,0 +1,21 @@ +import { join } from 'node:path'; + +export const JS_BENCHMARKING_PLUGIN_SLUG = 'js-benchmarking'; + +function withRunnerRoot(runnerName: string): string { + // @TODO replace with `@code-pushup/js-benchmarking-plugin/src/runner/` + return join( + 'dist', + 'examples', + 'plugins', + `${JS_BENCHMARKING_PLUGIN_SLUG}.${runnerName}.runner.js`, + ); +} + +export const JS_BENCHMARKING_TINYBENCH_RUNNER_PATH = + withRunnerRoot('tinybench'); +export const JS_BENCHMARKING_BENCHMARK_RUNNER_PATH = + withRunnerRoot('benchmark'); +export const JS_BENCHMARKING_BENNY_RUNNER_PATH = withRunnerRoot('benny'); +export const JS_BENCHMARKING_DEFAULT_RUNNER_PATH = + JS_BENCHMARKING_TINYBENCH_RUNNER_PATH; diff --git a/examples/plugins/src/js-benchmarking/src/index.ts b/examples/plugins/src/js-benchmarking/src/index.ts new file mode 100644 index 000000000..6f49162dd --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/index.ts @@ -0,0 +1,11 @@ +export { + JS_BENCHMARKING_PLUGIN_SLUG, + JS_BENCHMARKING_BENCHMARK_RUNNER_PATH, + JS_BENCHMARKING_TINYBENCH_RUNNER_PATH, + JS_BENCHMARKING_BENNY_RUNNER_PATH, + JS_BENCHMARKING_DEFAULT_RUNNER_PATH, +} from './constants'; +export type { BenchmarkResult, SuiteConfig, BenchmarkRunner } from './runner'; +export { JsBenchmarkingPluginConfig } from './config'; +export { jsBenchmarkingPlugin } from './js-benchmarking.plugin'; +export { jsBenchmarkingSuiteNameToCategoryRef } from './utils'; diff --git a/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.integration.test.ts b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.integration.test.ts new file mode 100644 index 000000000..b1bb0f2d0 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.integration.test.ts @@ -0,0 +1,42 @@ +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect } from 'vitest'; +import { executePlugin } from '@code-pushup/core'; +import { PluginConfig, pluginConfigSchema } from '@code-pushup/models'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from './constants'; +import { jsBenchmarkingPlugin } from './js-benchmarking.plugin'; + +const targetPath = join( + fileURLToPath(dirname(import.meta.url)), + '..', + '..', + '..', + '..', + 'perf', + 'dummy-suite', + 'index.ts', +); + +describe('jsBenchmarkingPlugin-execution', () => { + // @TODO move to e2e tests when plugin is released officially + // eslint-disable-next-line vitest/no-disabled-tests + it.skip('should execute', async () => { + const pluginConfig = await jsBenchmarkingPlugin({ + targets: [targetPath], + }); + expect(() => pluginConfigSchema.parse(pluginConfig)).not.toThrow(); + await expect(executePlugin(pluginConfig)).resolves.toEqual( + expect.objectContaining({ + slug: JS_BENCHMARKING_PLUGIN_SLUG, + title: 'JS Benchmarking', + icon: 'folder-benchmark', + audits: [ + { + slug: `${JS_BENCHMARKING_PLUGIN_SLUG}-suite-1`, + title: 'dummy-suite', + }, + ], + } satisfies Omit), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.ts b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.ts new file mode 100644 index 000000000..5e652dea8 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.ts @@ -0,0 +1,39 @@ +import type { PluginConfig } from '@code-pushup/models'; +import { ensureDirectoryExists } from '@code-pushup/utils'; +import { jsBenchmarkingPluginOptionsSchema } from './config'; +import { + JS_BENCHMARKING_DEFAULT_RUNNER_PATH, + JS_BENCHMARKING_PLUGIN_SLUG, +} from './constants'; +import { createRunnerFunction } from './runner'; +import type { BenchmarkRunnerOptions } from './runner/types'; +import { type LoadOptions, loadSuites, toAuditMetadata } from './utils'; + +export type PluginOptions = { + targets: string[]; + runnerPath?: string; +} & LoadOptions & + BenchmarkRunnerOptions; + +export async function jsBenchmarkingPlugin( + options: PluginOptions, +): Promise { + const { + tsconfig, + targets, + outputDir = '.code-pushup', + runnerPath = JS_BENCHMARKING_DEFAULT_RUNNER_PATH, + } = jsBenchmarkingPluginOptionsSchema.parse(options); + + await ensureDirectoryExists(outputDir); + // load the suites at before returning the plugin config to be able to return a more dynamic config + const suites = await loadSuites(targets, { tsconfig }); + + return { + slug: JS_BENCHMARKING_PLUGIN_SLUG, + title: 'JS Benchmarking', + icon: 'folder-benchmark', + audits: toAuditMetadata(suites.map(({ suiteName }) => suiteName)), + runner: createRunnerFunction(suites, { outputDir, runnerPath }), + } satisfies PluginConfig; +} diff --git a/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.unit.test.ts b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.unit.test.ts new file mode 100644 index 000000000..680ff906d --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/js-benchmarking.plugin.unit.test.ts @@ -0,0 +1,51 @@ +import { describe, expect } from 'vitest'; +import { PluginConfig, pluginConfigSchema } from '@code-pushup/models'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from './constants'; +import { jsBenchmarkingPlugin } from './js-benchmarking.plugin'; +import { BenchmarkResult } from './runner/types'; + +vi.mock('./utils', async () => { + const all: object = await vi.importActual('./utils'); + return { + ...all, + loadSuites: vi.fn().mockImplementation((suiteNames: string[]) => + suiteNames.map( + (suiteName, index) => + ({ + suiteName: suiteName.replace('.ts', ''), + name: + index === 0 + ? 'current-implementation' + : `implementation-${index}`, + rme: index === 0 ? 1 : Math.random(), + hz: index === 0 ? 1 : Math.random(), + isFastest: index === 0, + isTarget: index === 0, + samples: suiteNames.length * 10, + } satisfies BenchmarkResult), + ), + ), + }; +}); + +describe('jsBenchmarkingPlugin-config-creation', () => { + it('should create valid config', async () => { + const pluginConfig = await jsBenchmarkingPlugin({ + targets: ['dummy-suite.ts'], + }); + expect(() => pluginConfigSchema.parse(pluginConfig)).not.toThrow(); + expect(pluginConfig).toEqual( + expect.objectContaining({ + slug: JS_BENCHMARKING_PLUGIN_SLUG, + title: 'JS Benchmarking', + icon: 'folder-benchmark', + audits: [ + { + slug: `${JS_BENCHMARKING_PLUGIN_SLUG}-dummy-suite`, + title: 'dummy-suite', + }, + ], + } satisfies Omit), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.integration.test.ts b/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.integration.test.ts new file mode 100644 index 000000000..6da58660e --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.integration.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; +import dummySuite from '../../../../perf/dummy-suite'; +import { benchmarkRunner } from './benchmark.suite-runner'; + +describe('benchmarkRunner-execution', () => { + // @TODO move to e2e tests when plugin is released officially + // eslint-disable-next-line vitest/no-disabled-tests + it.skip('should execute valid suite', async () => { + await expect(benchmarkRunner.run(dummySuite)).resolves.toStrictEqual( + expect.arrayContaining([ + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-1', + isTarget: true, + hz: expect.any(Number), + isFastest: true, + rme: expect.any(Number), + samples: expect.any(Number), + }), + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-2', + isTarget: false, + hz: expect.any(Number), + isFastest: false, + rme: expect.any(Number), + samples: expect.any(Number), + }), + ]), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.ts b/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.ts new file mode 100644 index 000000000..ab807a53c --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/benchmark.suite-runner.ts @@ -0,0 +1,86 @@ +import Benchmark, { Event, type Suite, type Target } from 'benchmark'; +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import type { + BenchmarkResult, + BenchmarkRunner, + BenchmarkRunnerOptions, + SuiteConfig, +} from './types'; + +export const benchmarkRunner = { + run: async ( + { suiteName, cases, targetImplementation }: SuiteConfig, + options: BenchmarkRunnerOptions = {}, + ): Promise => { + const { + verbose = false, + outputFileName: fileName = 'benchmark-report', + outputDir: folder = JS_BENCHMARKING_PLUGIN_SLUG, + } = options; + + return new Promise((resolve, reject) => { + // This is not working with named imports + // eslint-disable-next-line import/no-named-as-default-member + const suite = new Benchmark.Suite(suiteName); + + // Add Listener + Object.entries({ + error: (e: { target?: { error?: unknown } }) => { + reject(e.target?.error ?? e); + }, + cycle: function (event: Event) { + if (verbose) { + // @TODO use cliui.logger.info(String(event.target)) + // eslint-disable-next-line no-console + console.log(String(event.target)); + } + }, + complete: () => { + const result = benchToBenchmarkResult(suite, { + suiteName, + cases, + targetImplementation, + }); + if (fileName || folder) { + void writeFile( + join(folder, `${fileName}.json`), + JSON.stringify(result, null, 2), + ).then(() => { + resolve(result); + }); + } else { + resolve(result); + } + }, + }).forEach(([name, fn]) => suite.on(name, fn)); + + // register test cases + cases.forEach(tuple => suite.add(...tuple)); + + suite.run({ async: true }); + }); + }, +} satisfies BenchmarkRunner; + +export function benchToBenchmarkResult( + suite: Suite, + { targetImplementation, suiteName }: SuiteConfig, +): BenchmarkResult[] { + const fastest = String(suite.filter('fastest').map('name')[0]); + return suite.map( + (bench: Target) => + ({ + suiteName, + name: bench.name || '', + hz: bench.hz ?? 0, // operations per second + rme: bench.stats?.rme ?? 0, // relative margin of error + samples: bench.stats?.sample.length ?? 0, // number of samples + isFastest: fastest === bench.name, + isTarget: targetImplementation === bench.name, + } satisfies BenchmarkResult), + ) as BenchmarkResult[]; // suite.map has a broken typing +} + +export default benchmarkRunner; diff --git a/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.integration.test.ts b/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.integration.test.ts new file mode 100644 index 000000000..312db54da --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.integration.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from 'vitest'; +import dummySuite from '../../../../perf/dummy-suite'; +import { bennyRunner } from './benny.suite-runner'; + +describe('bennyRunner-execution', () => { + // @TODO move to e2e tests when plugin is released officially + + it('should execute valid suite', async () => { + await expect(bennyRunner.run(dummySuite)).resolves.toStrictEqual( + expect.arrayContaining([ + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-1', + isTarget: true, + hz: expect.any(Number), + isFastest: true, + rme: expect.any(Number), + samples: expect.any(Number), + }), + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-2', + isTarget: false, + hz: expect.any(Number), + isFastest: false, + rme: expect.any(Number), + samples: expect.any(Number), + }), + ]), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.ts b/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.ts new file mode 100644 index 000000000..ba9df5ef4 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/benny.suite-runner.ts @@ -0,0 +1,59 @@ +import benny from 'benny'; +import type { Summary } from 'benny/lib/internal/common-types'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import { BenchmarkResult, BenchmarkRunnerOptions, SuiteConfig } from './types'; + +export const bennyRunner = { + run: async ( + { suiteName, cases, targetImplementation }: SuiteConfig, + options: BenchmarkRunnerOptions = {}, + ): Promise => { + const { + outputFileName: file = 'benny-report', + outputDir: folder = JS_BENCHMARKING_PLUGIN_SLUG, + } = options; + + return new Promise(resolve => { + // This is not working with named imports + void benny.suite( + suiteName, + ...cases.map(([name, fn]) => + benny.add(name, () => { + fn(); + }), + ), + + benny.cycle(), + + benny.complete(summary => { + resolve( + benchToBenchmarkResult(summary, { + suiteName, + cases, + targetImplementation, + }), + ); + }), + benny.save({ file, folder, format: 'json', details: true }), + ); + }); + }, +}; + +export function benchToBenchmarkResult( + suite: Summary, + { targetImplementation, suiteName }: SuiteConfig, +) { + const { name: fastestName } = suite.fastest; + return suite.results.map(({ ops, name: caseName, details }) => ({ + suiteName, + name: caseName || '', + hz: ops, // operations per second + rme: details.relativeMarginOfError, // relative margin of error + samples: details.sampleResults.length, // samples recorded for this case + isFastest: fastestName === caseName, + isTarget: targetImplementation === caseName, + })); +} + +export default bennyRunner; diff --git a/examples/plugins/src/js-benchmarking/src/runner/index.ts b/examples/plugins/src/js-benchmarking/src/runner/index.ts new file mode 100644 index 000000000..f5ce320f4 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/index.ts @@ -0,0 +1,3 @@ +export type { SuiteConfig, BenchmarkRunner, BenchmarkResult } from './types'; +export { createRunnerFunction } from './runner'; +export { toAuditSlug } from './utils'; diff --git a/examples/plugins/src/js-benchmarking/src/runner/runner.ts b/examples/plugins/src/js-benchmarking/src/runner/runner.ts new file mode 100644 index 000000000..261e6fddd --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/runner.ts @@ -0,0 +1,42 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { AuditOutputs, RunnerFunction } from '@code-pushup/models'; +import { importEsmModule } from '@code-pushup/utils'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import { BenchmarkResult, BenchmarkRunner, SuiteConfig } from './types'; +import { suiteResultToAuditOutput } from './utils'; + +export function createRunnerFunction( + suites: SuiteConfig[], + options: { + runnerPath: string; + outputDir?: string; + }, +): RunnerFunction { + const { + outputDir = join('.code-pushup', JS_BENCHMARKING_PLUGIN_SLUG), + runnerPath: filepath, + } = options; + return async (): Promise => { + const allSuiteResults: BenchmarkResult[][] = []; + // Execute each suite sequentially + // eslint-disable-next-line functional/no-loop-statements + for (const suite of suites) { + const runner = await importEsmModule({ + filepath, + }); + const result: BenchmarkResult[] = await runner.run(suite); + if (outputDir && outputDir !== '') { + await writeFile( + join(outputDir, `${suite.suiteName}-benchmark.json`), + JSON.stringify(result, null, 2), + ); + } + // eslint-disable-next-line functional/immutable-data + allSuiteResults.push(result); + } + + // create audit output + return allSuiteResults.map(results => suiteResultToAuditOutput(results)); + }; +} diff --git a/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.integration.test.ts b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.integration.test.ts new file mode 100644 index 000000000..095750f43 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.integration.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import dummySuite from '../../../../perf/dummy-suite'; +import { tinybenchRunner } from './tinybench.suite-runner'; + +describe('tinybenchRunner-execution', () => { + it('should execute valid suite', async () => { + await expect(tinybenchRunner.run(dummySuite)).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-1', + isTarget: true, + hz: expect.any(Number), + isFastest: true, + rme: expect.any(Number), + samples: expect.any(Number), + }), + expect.objectContaining({ + suiteName: 'dummy-suite', + name: 'case-2', + isTarget: false, + hz: expect.any(Number), + isFastest: false, + rme: expect.any(Number), + samples: expect.any(Number), + }), + ]), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.ts b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.ts new file mode 100644 index 000000000..3b8da80e0 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.ts @@ -0,0 +1,82 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Bench } from 'tinybench'; +import { ensureDirectoryExists } from '@code-pushup/utils'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import type { + BenchmarkResult, + BenchmarkRunner, + BenchmarkRunnerOptions, + SuiteConfig, +} from './types'; + +export const tinybenchRunner = { + run: async ( + { suiteName, cases, targetImplementation, time = 3000 }: SuiteConfig, + options: BenchmarkRunnerOptions = {}, + ): Promise => { + const { + outputFileName: fileName = 'tinybench-report', + outputDir: folder = JS_BENCHMARKING_PLUGIN_SLUG, + } = options; + const suite = new Bench({ time }); + + // register test cases + cases.forEach(tuple => suite.add(...tuple)); + + await suite.warmup(); // make results more reliable, ref: https://github.com/tinylibs/tinybench/pull/50 + await suite.run(); + + const result = benchToBenchmarkResult(suite, { + suiteName, + cases, + targetImplementation, + time, + }); + + if (fileName || folder) { + await ensureDirectoryExists(folder); + return writeFile( + join(folder, `${fileName}.json`), + JSON.stringify(result, null, 2), + ).then(() => result); + } + + return result; + }, +} satisfies BenchmarkRunner; + +export function benchToBenchmarkResult( + bench: Bench, + suite: SuiteConfig, +): BenchmarkResult[] { + const { suiteName, cases, targetImplementation } = suite; + const caseNames = cases.map(([name]) => name); + const results = caseNames + .map(caseName => { + const result = bench.getTask(caseName)?.result ?? { + hz: 0, + rme: 0, + samples: [], + }; + return { + suiteName, + name: caseName, + hz: result.hz, + rme: result.rme, + samples: result.samples.length, + isTarget: targetImplementation === caseName, + isFastest: false, // preliminary result + } satisfies BenchmarkResult; + }) + // sort by hz to get fastest at the top + .sort(({ hz: hzA }, { hz: hzB }) => hzA - hzB); + + return results.map(result => + results.at(1)?.name === result.name + ? { ...result, isFastest: true } + : result, + ); +} + +export default tinybenchRunner; diff --git a/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.unit.test.ts b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.unit.test.ts new file mode 100644 index 000000000..5a9f617ff --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/tinybench.suite-runner.unit.test.ts @@ -0,0 +1,58 @@ +import { Bench } from 'tinybench'; +import { describe, expect, it } from 'vitest'; +import { benchToBenchmarkResult } from './tinybench.suite-runner'; + +describe('benchToBenchmarkResult', () => { + it('should transform a tinybench Bench to a enriched BenchmarkResult', () => { + const resultMap = { + 'current-implementation': { + hz: 175.333_33, + rme: 0.444_44, + samples: [5.6, 5.6], + }, + 'slower-implementation': { + hz: 75.333_33, + rme: 0.444_44, + samples: [5.6666, 5.6666], + }, + }; + const suitNames = Object.keys(resultMap); + + expect( + benchToBenchmarkResult( + { + getTask: (name: keyof typeof resultMap) => ({ + result: resultMap[name], + }), + results: Object.values(resultMap), + } as Bench, + { + suiteName: 'suite-1', + cases: suitNames.map(name => [name, vi.fn()]), + targetImplementation: suitNames.at(0) as string, + }, + ), + ).toStrictEqual( + expect.arrayContaining([ + expect.objectContaining({ + suiteName: 'suite-1', + name: 'current-implementation', + isTarget: true, + hz: 175.333_33, + isFastest: true, + rme: 0.444_44, + samples: 2, + }), + expect.objectContaining({ + suiteName: 'suite-1', + name: 'slower-implementation', + isTarget: false, + hz: 75.333_33, + isFastest: false, + rme: 0.444_44, + samples: 2, + }), + ]), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/runner/types.ts b/examples/plugins/src/js-benchmarking/src/runner/types.ts new file mode 100644 index 000000000..56d08f719 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/types.ts @@ -0,0 +1,30 @@ +export type SuiteConfig = { + suiteName: string; + targetImplementation: string; + // eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents + cases: [string, (...args: unknown[]) => Promise | unknown][]; + time?: number; +}; + +export type BenchmarkResult = { + hz: number; + rme: number; + suiteName: string; + name: string; + isFastest: boolean; + isTarget: boolean; + // not given in all benchmark implementations + samples?: number; +}; + +export type BenchmarkRunnerOptions = { + verbose?: boolean; + outputDir?: string; + outputFileName?: string; +}; +export type BenchmarkRunner = { + run: ( + config: SuiteConfig, + options?: BenchmarkRunnerOptions, + ) => Promise; +}; diff --git a/examples/plugins/src/js-benchmarking/src/runner/utils.ts b/examples/plugins/src/js-benchmarking/src/runner/utils.ts new file mode 100644 index 000000000..d67b141b6 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/utils.ts @@ -0,0 +1,47 @@ +import { AuditOutput, Issue } from '@code-pushup/models'; +import { slugify } from '@code-pushup/utils'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import { BenchmarkResult } from './types'; + +export function toAuditSlug(suiteName: string): string { + return `${JS_BENCHMARKING_PLUGIN_SLUG}-${slugify(suiteName)}`; +} + +/** + * scoring of js computation time can be used in 2 ways: + * - many implementations against the current implementation to maintain the fastest (score is 100 based on fastest) + * - testing many implementations/libs to pick the fastest + * @param results + */ +export function suiteResultToAuditOutput( + results: BenchmarkResult[], +): AuditOutput { + const { hz: maxHz, suiteName } = results.find( + ({ isFastest }) => isFastest, + ) as BenchmarkResult; + const { hz: targetHz } = results.find( + ({ isTarget }) => isTarget, + ) as BenchmarkResult; + + return { + slug: toAuditSlug(suiteName), + displayValue: `${targetHz.toFixed(2)} ops/sec`, + score: targetHz <= maxHz ? targetHz / maxHz : 1, + value: Number.parseInt(targetHz.toString(), 10), + details: { + issues: results.map(({ name, hz, rme, samples, isTarget, isFastest }) => { + const targetIcon = isTarget ? '🎯' : ''; + const postfix = isFastest + ? '(fastest 🔥)' + : `(${((1 - hz / maxHz) * 100).toFixed(1)}% slower)`; + return { + // fast-glob x 40,824 ops/sec ±4.44% (85 runs sampled) + message: `${targetIcon}${name} x ${hz.toFixed( + 2, + )} ops/sec ±${rme.toFixed(2)}; ${samples} samples ${postfix}`, + severity: hz < maxHz && isTarget ? 'error' : 'info', + } satisfies Issue; + }), + }, + }; +} diff --git a/examples/plugins/src/js-benchmarking/src/runner/utils.unit.test.ts b/examples/plugins/src/js-benchmarking/src/runner/utils.unit.test.ts new file mode 100644 index 000000000..2e7df05ef --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/runner/utils.unit.test.ts @@ -0,0 +1,264 @@ +import { describe, expect, it } from 'vitest'; +import { auditOutputSchema } from '@code-pushup/models'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from '../constants'; +import { BenchmarkResult } from './types'; +import { suiteResultToAuditOutput, toAuditSlug } from './utils'; + +describe('toAuditSlug', () => { + it('should create slug string', () => { + expect(toAuditSlug('glob')).toBe(`${JS_BENCHMARKING_PLUGIN_SLUG}-glob`); + }); +}); + +describe('suiteResultToAuditOutput', () => { + it('should produce valid minimal AuditOutput for a single result', () => { + const auditOutput = suiteResultToAuditOutput([ + { + suiteName: 'sort', + hz: 100, + rme: 1, + name: 'implementation-1', + isFastest: true, + isTarget: true, + samples: 4, + }, + ]); + expect(auditOutput).toEqual( + expect.objectContaining({ + slug: toAuditSlug('sort'), + score: 1, + value: 100, + displayValue: '100.00 ops/sec', + }), + ); + expect(() => auditOutputSchema.parse(auditOutput)).not.toThrow(); + }); + + it('should have hz as value and converted to integer', () => { + expect( + suiteResultToAuditOutput([ + { + hz: 100.1111, + isFastest: true, + isTarget: true, + suiteName: 'sort', + rme: 1, + } as BenchmarkResult, + ]), + ).toEqual(expect.objectContaining({ value: 100 })); + }); + + it('should score based on maxHz', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'glob', + hz: 100, + rme: 2.5, + name: 'globby', + isFastest: true, + isTarget: false, + samples: 4, + }, + { + suiteName: 'glob', + hz: 10, + rme: 2.5, + name: 'globby2', + isFastest: false, + isTarget: true, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + score: 0.1, + }), + ); + }); + + it('should score a maximum of 1', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'glob', + hz: 0.1, + rme: 2.5, + name: 'target', + isFastest: false, + isTarget: true, + samples: 4, + }, + { + suiteName: 'glob', + hz: 1, + rme: 2.5, + name: 'other', + isFastest: true, + isTarget: false, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + score: 0.1, + }), + ); + }); + + it('should format value to 2 floating positions', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'glob', + hz: 1.111_111, + rme: 2.5, + name: 'globby', + isFastest: true, + isTarget: true, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + displayValue: '1.11 ops/sec', + }), + ); + }); + + it('should pick fastest test result as scoring base', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'sort', + hz: 100, + rme: 1, + name: 'implementation-1', + isFastest: true, + isTarget: false, + samples: 4, + }, + { + suiteName: 'sort', + hz: 10, + rme: 1, + name: 'implementation-2', + isFastest: false, + isTarget: true, + samples: 4, + }, + ]), + ).toEqual(expect.objectContaining({ score: 0.1 })); + }); + + it('should pick target test result for AuditOutput data', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'sort', + hz: 99, + rme: 1, + name: 'implementation-1', + isFastest: true, + isTarget: true, + samples: 4, + }, + { + suiteName: 'sort', + hz: 10, + rme: 1, + name: 'implementation-2', + isFastest: false, + isTarget: false, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + slug: toAuditSlug('sort'), + value: 99, + displayValue: '99.00 ops/sec', + }), + ); + }); + + it('should have correct details for a suit with score 100', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'sort', + hz: 100, + rme: 1, + name: 'implementation-1', + isFastest: true, + isTarget: true, + samples: 5, + }, + { + suiteName: 'sort', + hz: 60, + rme: 1.12, + name: 'implementation-2', + isFastest: false, + isTarget: false, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + details: { + issues: expect.arrayContaining([ + { + message: `🎯implementation-1 x 100.00 ops/sec ±1.00; 5 samples (fastest 🔥)`, + severity: 'info', + }, + { + message: `implementation-2 x 60.00 ops/sec ±1.12; 4 samples (40.0% slower)`, + severity: 'info', + }, + ]), + }, + }), + ); + }); + + it('should have correct details for a suit with score long floating number', () => { + expect( + suiteResultToAuditOutput([ + { + suiteName: 'sort', + hz: 100.0001, + rme: 1, + name: 'implementation-1', + isFastest: true, + isTarget: false, + samples: 5, + }, + { + suiteName: 'sort', + hz: 60.123, + rme: 1.12, + name: 'implementation-2', + isFastest: false, + isTarget: true, + samples: 4, + }, + ]), + ).toEqual( + expect.objectContaining({ + details: { + issues: expect.arrayContaining([ + { + message: `implementation-1 x 100.00 ops/sec ±1.00; 5 samples (fastest 🔥)`, + severity: 'info', + }, + { + message: `🎯implementation-2 x 60.12 ops/sec ±1.12; 4 samples (39.9% slower)`, + severity: 'error', + }, + ]), + }, + }), + ); + }); +}); diff --git a/examples/plugins/src/js-benchmarking/src/utils.ts b/examples/plugins/src/js-benchmarking/src/utils.ts new file mode 100644 index 000000000..44a4b5b61 --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/utils.ts @@ -0,0 +1,47 @@ +import { Audit, type CategoryRef } from '@code-pushup/models'; +import { importEsmModule } from '@code-pushup/utils'; +import { JS_BENCHMARKING_PLUGIN_SLUG } from './constants'; +import { type SuiteConfig, toAuditSlug } from './runner'; + +export function toAuditTitle(suiteName: string): string { + return `${suiteName}`; +} + +export function toAuditMetadata(suiteNames: string[]): Audit[] { + return suiteNames.map( + suiteName => + ({ + slug: toAuditSlug(suiteName), + title: toAuditTitle(suiteName), + } satisfies Audit), + ); +} +export function jsBenchmarkingSuiteNameToCategoryRef( + suiteName: string, +): CategoryRef { + return { + type: 'audit', + plugin: JS_BENCHMARKING_PLUGIN_SLUG, + slug: toAuditSlug(suiteName), + weight: 1, + } satisfies CategoryRef; +} + +export type LoadOptions = { + tsconfig?: string; +}; + +export function loadSuites( + targets: string[], + options: LoadOptions = {}, +): Promise { + const { tsconfig } = options; + return Promise.all( + targets.map((filepath: string) => + importEsmModule({ + tsconfig, + filepath, + }), + ), + ); +} diff --git a/examples/plugins/src/js-benchmarking/src/utils.unit.test.ts b/examples/plugins/src/js-benchmarking/src/utils.unit.test.ts new file mode 100644 index 000000000..0cb98f69a --- /dev/null +++ b/examples/plugins/src/js-benchmarking/src/utils.unit.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, it } from 'vitest'; +import { SuiteConfig } from './runner/types'; +import { toAuditSlug } from './runner/utils'; +import { + jsBenchmarkingSuiteNameToCategoryRef, + loadSuites, + toAuditMetadata, + toAuditTitle, +} from './utils'; + +vi.mock('@code-pushup/utils', async () => { + const actual = await vi.importActual('@code-pushup/utils'); + + return { + ...actual, + importEsmModule: vi.fn().mockImplementation( + ({ filepath = '' }: { filepath: string }) => + ({ + suiteName: filepath.replace('.ts', ''), + targetImplementation: 'current-implementation', + cases: [ + ['current-implementation', vi.fn()], + ['slower-implementation', vi.fn()], + ], + } satisfies SuiteConfig), + ), + }; +}); + +describe('toAuditTitle', () => { + it('should create title string', () => { + expect(toAuditTitle('glob')).toBe('glob'); + }); +}); + +describe('toAuditMetadata', () => { + it('should create metadata string', () => { + expect(toAuditMetadata(['glob'])).toStrictEqual([ + { + slug: toAuditSlug('glob'), + title: toAuditTitle('glob'), + }, + ]); + }); +}); + +describe('jsBenchmarkingSuiteNameToCategoryRef', () => { + it('should create a valid CategoryRef form suiteName', () => { + expect(jsBenchmarkingSuiteNameToCategoryRef('glob')).toEqual({ + slug: toAuditSlug('glob'), + type: 'audit', + weight: 1, + plugin: 'js-benchmarking', + }); + }); +}); + +describe('loadSuites', () => { + it('should load given suites', async () => { + await expect(loadSuites(['suite-1.ts', 'suite-2.ts'])).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ suiteName: 'suite-1' }), + expect.objectContaining({ suiteName: 'suite-2' }), + ]), + ); + }); +}); diff --git a/examples/plugins/tsconfig.lib.json b/examples/plugins/tsconfig.lib.json index ef2f7e2b3..f9be5a838 100644 --- a/examples/plugins/tsconfig.lib.json +++ b/examples/plugins/tsconfig.lib.json @@ -11,6 +11,7 @@ "vite.config.integration.ts", "src/**/*.test.ts", "src/**/*.mock.ts", - "mocks/**/*.ts" + "mocks/**/*.ts", + "**/perf/**/*.ts" ] } diff --git a/examples/plugins/tsconfig.perf.json b/examples/plugins/tsconfig.perf.json new file mode 100644 index 000000000..a091eb002 --- /dev/null +++ b/examples/plugins/tsconfig.perf.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "../../dist/out-tsc", + "declaration": true, + "types": ["node"] + }, + "include": ["src/**/*.ts", "perf/**/*.ts", "code-pushup.config.ts"], + "exclude": [ + "vite.config.unit.ts", + "vite.config.integration.ts", + "src/**/*.test.ts", + "src/**/*.mock.ts", + "mocks/**/*.ts" + ] +} diff --git a/package-lock.json b/package-lock.json index 40397d8e0..8315d034e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -57,6 +57,7 @@ "@vitest/coverage-v8": "1.3.1", "@vitest/ui": "1.3.1", "benchmark": "^2.1.4", + "benny": "^3.7.1", "commitizen": "^4.3.0", "commitlint-plugin-tense": "^1.0.2", "conventional-changelog-angular": "^7.0.0", @@ -129,6 +130,48 @@ "node": ">=6.0.0" } }, + "node_modules/@arrows/array": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@arrows/array/-/array-1.4.1.tgz", + "integrity": "sha512-MGYS8xi3c4tTy1ivhrVntFvufoNzje0PchjEz6G/SsWRgUKxL4tKwS6iPdO8vsaJYldagAeWMd5KRD0aX3Q39g==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.2.2" + } + }, + "node_modules/@arrows/composition": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/@arrows/composition/-/composition-1.2.2.tgz", + "integrity": "sha512-9fh1yHwrx32lundiB3SlZ/VwuStPB4QakPsSLrGJFH6rCXvdrd060ivAZ7/2vlqPnEjBkPRRXOcG1YOu19p2GQ==", + "dev": true + }, + "node_modules/@arrows/dispatch": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@arrows/dispatch/-/dispatch-1.0.3.tgz", + "integrity": "sha512-v/HwvrFonitYZM2PmBlAlCqVqxrkIIoiEuy5bQgn0BdfvlL0ooSBzcPzTMrtzY8eYktPyYcHg8fLbSgyybXEqw==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.2.2" + } + }, + "node_modules/@arrows/error": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@arrows/error/-/error-1.0.2.tgz", + "integrity": "sha512-yvkiv1ay4Z3+Z6oQsUkedsQm5aFdyPpkBUQs8vejazU/RmANABx6bMMcBPPHI4aW43VPQmXFfBzr/4FExwWTEA==", + "dev": true + }, + "node_modules/@arrows/multimethod": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/@arrows/multimethod/-/multimethod-1.4.1.tgz", + "integrity": "sha512-AZnAay0dgPnCJxn3We5uKiB88VL+1ZIF2SjZohLj6vqY2UyvB/sKdDnFP+LZNVsTC5lcnGPmLlRRkAh4sXkXsQ==", + "dev": true, + "dependencies": { + "@arrows/array": "^1.4.1", + "@arrows/composition": "^1.2.2", + "@arrows/error": "^1.0.2", + "fast-deep-equal": "^3.1.3" + } + }, "node_modules/@babel/code-frame": { "version": "7.24.2", "dev": true, @@ -8324,6 +8367,15 @@ "dev": true, "license": "ISC" }, + "node_modules/astral-regex": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", + "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/async": { "version": "3.2.5", "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", @@ -8792,6 +8844,163 @@ "platform": "^1.3.3" } }, + "node_modules/benny": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/benny/-/benny-3.7.1.tgz", + "integrity": "sha512-USzYxODdVfOS7JuQq/L0naxB788dWCiUgUTxvN+WLPt/JfcDURNNj8kN/N+uK6PDvuR67/9/55cVKGPleFQINA==", + "dev": true, + "dependencies": { + "@arrows/composition": "^1.0.0", + "@arrows/dispatch": "^1.0.2", + "@arrows/multimethod": "^1.1.6", + "benchmark": "^2.1.4", + "common-tags": "^1.8.0", + "fs-extra": "^10.0.0", + "json2csv": "^5.0.6", + "kleur": "^4.1.4", + "log-update": "^4.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/benny/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/benny/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/benny/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/benny/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "dev": true + }, + "node_modules/benny/node_modules/fs-extra": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-10.1.0.tgz", + "integrity": "sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/benny/node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "dev": true, + "engines": { + "node": ">=8" + } + }, + "node_modules/benny/node_modules/log-update": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/log-update/-/log-update-4.0.0.tgz", + "integrity": "sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==", + "dev": true, + "dependencies": { + "ansi-escapes": "^4.3.0", + "cli-cursor": "^3.1.0", + "slice-ansi": "^4.0.0", + "wrap-ansi": "^6.2.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/benny/node_modules/slice-ansi": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", + "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "astral-regex": "^2.0.0", + "is-fullwidth-code-point": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/benny/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/benny/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/benny/node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/big.js": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", @@ -10056,6 +10265,15 @@ "node": ">=v12" } }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/compare-func": { "version": "2.0.0", "dev": true, @@ -17340,6 +17558,34 @@ "dev": true, "license": "ISC" }, + "node_modules/json2csv": { + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/json2csv/-/json2csv-5.0.7.tgz", + "integrity": "sha512-YRZbUnyaJZLZUJSRi2G/MqahCyRv9n/ds+4oIetjDF3jWQA7AG7iSeKTiZiCNqtMZM7HDyt0e/W6lEnoGEmMGA==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true, + "dependencies": { + "commander": "^6.1.0", + "jsonparse": "^1.3.1", + "lodash.get": "^4.4.2" + }, + "bin": { + "json2csv": "bin/json2csv.js" + }, + "engines": { + "node": ">= 10", + "npm": ">= 6.13.0" + } + }, + "node_modules/json2csv/node_modules/commander": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", + "integrity": "sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==", + "dev": true, + "engines": { + "node": ">= 6" + } + }, "node_modules/json5": { "version": "2.2.3", "dev": true, @@ -17781,6 +18027,12 @@ "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", "dev": true }, + "node_modules/lodash.get": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", + "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", + "dev": true + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", diff --git a/package.json b/package.json index 6de5fbb4d..0b478d714 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,7 @@ "@vitest/coverage-v8": "1.3.1", "@vitest/ui": "1.3.1", "benchmark": "^2.1.4", + "benny": "^3.7.1", "commitizen": "^4.3.0", "commitlint-plugin-tense": "^1.0.2", "conventional-changelog-angular": "^7.0.0", diff --git a/packages/cli/README.md b/packages/cli/README.md index 89574ac4f..5f0489c77 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -171,6 +171,7 @@ Each example is fully tested to demonstrate best practices for plugin testing as - 📏 [File Size](../../examples/plugins/src/file-size) - example of basic runner executor - 📦 [Package Json](../../examples/plugins/src/package-json) - example of audits and groups - 🔥 [Lighthouse](../../examples/plugins/src/lighthouse) (official implementation [here](../../../../packages/plugin-lighthouse)) - example of a basic command executor +- 📊 [Benchmark JS](../../examples/plugins/src/benchmark-js) - js micro benchmarking reports of different packages ## CLI commands and options diff --git a/packages/core/src/lib/implementation/read-rc-file.integration.test.ts b/packages/core/src/lib/implementation/read-rc-file.integration.test.ts index f3e470de8..4e75046dc 100644 --- a/packages/core/src/lib/implementation/read-rc-file.integration.test.ts +++ b/packages/core/src/lib/implementation/read-rc-file.integration.test.ts @@ -1,4 +1,5 @@ import { dirname, join } from 'node:path'; +import * as process from 'node:process'; import { fileURLToPath } from 'node:url'; import { describe, expect } from 'vitest'; import { readRcByPath } from './read-rc-file'; @@ -66,6 +67,24 @@ describe('readRcByPath', () => { ).rejects.toThrow(/Provided path .* is not valid./); }); + it('should throw the configuration using a tsconfig path that does not exist', async () => { + await expect( + readRcByPath( + join(configDirPath, 'code-pushup.needs-tsconfig.config.ts'), + 'tsconfig.wrong.json', + ), + ).rejects.toThrow('tsconfig.wrong.json'); + }); + + it('should throw the configuration using a tsconfig path is not a file', async () => { + await expect( + readRcByPath( + join(configDirPath, 'code-pushup.needs-tsconfig.config.ts'), + process.cwd(), + ), + ).rejects.toThrow(`The tsconfig path '${process.cwd()}' is not a file`); + }); + it('should throw if the configuration is empty', async () => { await expect( readRcByPath(join(configDirPath, 'code-pushup.empty.config.js')), diff --git a/packages/core/src/lib/implementation/read-rc-file.ts b/packages/core/src/lib/implementation/read-rc-file.ts index 50a6370a9..d8fa53692 100644 --- a/packages/core/src/lib/implementation/read-rc-file.ts +++ b/packages/core/src/lib/implementation/read-rc-file.ts @@ -1,3 +1,4 @@ +import { stat } from 'node:fs/promises'; import { join } from 'node:path'; import { CONFIG_FILE_NAME, @@ -21,6 +22,13 @@ export async function readRcByPath( throw new Error('The path to the configuration file is empty.'); } + if (tsconfig) { + const s = await stat(tsconfig); + if (!s.isFile()) { + throw new Error(`The tsconfig path '${tsconfig}' is not a file.`); + } + } + if (!(await fileExists(filepath))) { throw new ConfigPathError(filepath); } diff --git a/packages/utils/perf/README.md b/packages/utils/perf/README.md index 32f5bea45..40a7abbea 100644 --- a/packages/utils/perf/README.md +++ b/packages/utils/perf/README.md @@ -1,9 +1,4 @@ # Micro Benchmarks -Execute any benchmark by running `nx run utils:perf` naming the folder e.g. `nx run utils:perf score-report`. - -To list benchmarks run `ls -l | grep "^d" | awk '{print $9}'` - -## scoreReport - -`nx run utils:perf score-report` +Execute any benchmark by running `nx run utils:perf --targets ` naming the entry point, e.g. `nx run utils:perf --targets score-report/index.ts`. +This script is documented [here](../../../tools/benchmark/docs/README.md) diff --git a/packages/utils/perf/crawl-file-system/fs-walk.ts b/packages/utils/perf/crawl-file-system/fs-walk.ts index cf07dd25e..74240731a 100644 --- a/packages/utils/perf/crawl-file-system/fs-walk.ts +++ b/packages/utils/perf/crawl-file-system/fs-walk.ts @@ -1,5 +1,5 @@ import { type Entry, walkStream } from '@nodelib/fs.walk'; -import { CrawlFileSystemOptions } from '../../src'; +import { type CrawlFileSystemOptions } from '../../src/lib/file-system'; // from https://github.com/webpro/knip/pull/426 export function crawlFileSystemFsWalk( diff --git a/packages/utils/perf/crawl-file-system/index.ts b/packages/utils/perf/crawl-file-system/index.ts index 4ebc378c5..1ad3d94d5 100644 --- a/packages/utils/perf/crawl-file-system/index.ts +++ b/packages/utils/perf/crawl-file-system/index.ts @@ -1,103 +1,88 @@ -import * as Benchmark from 'benchmark'; +import chalk from 'chalk'; import { join } from 'node:path'; -import { - CrawlFileSystemOptions, - crawlFileSystem, -} from '../../src/lib/file-system'; +import yargs from 'yargs'; +import type { CrawlFileSystemOptions } from '../../src/lib/file-system'; import { crawlFileSystemFsWalk } from './fs-walk'; -const PROCESS_ARGUMENT_TARGET_DIRECTORY = - process.argv - .find(arg => arg.startsWith('--directory')) - ?.split('=') - .at(-1) ?? ''; -const PROCESS_ARGUMENT_PATTERN = - process.argv - .find(arg => arg.startsWith('--pattern')) - ?.split('=') - .at(-1) ?? ''; - -const suite = new Benchmark.Suite('report-scoring'); - -const TARGET_DIRECTORY = - PROCESS_ARGUMENT_TARGET_DIRECTORY || - join(process.cwd(), '..', '..', '..', 'node_modules'); -const PATTERN = PROCESS_ARGUMENT_PATTERN || /.json$/; - -// ================== - -const start = performance.now(); - -// Add listener -const listeners = { - cycle: function (event: Benchmark.Event) { - console.info(String(event.target)); +const cli = yargs(process.argv).options({ + directory: { + type: 'string', + default: join('packages', 'utils'), }, - complete: () => { - if (typeof suite.filter === 'function') { - console.info(' '); - console.info( - `Total Duration: ${((performance.now() - start) / 1000).toFixed( - 2, - )} sec`, - ); - console.info(`Fastest is ${String(suite.filter('fastest').map('name'))}`); - } + pattern: { + type: 'string', + default: '.md', + }, + logs: { + type: 'boolean', + default: false, + }, + outputDir: { + type: 'string', + default: '.code-pushup', }, -}; - -// ================== - -// Add tests -const options = { - directory: TARGET_DIRECTORY, - pattern: PATTERN, -}; -suite.add('Base', wrapWithDefer(crawlFileSystem)); -suite.add('nodelib.fsWalk', wrapWithDefer(crawlFileSystemFsWalk)); - -// ================== - -// Add Listener -Object.entries(listeners).forEach(([name, fn]) => { - suite.on(name, fn); }); +const { directory, pattern, logs } = await cli.parseAsync(); -// ================== +if (logs) { + console.info( + 'You can adjust the test with the following arguments:' + + `directory target directory of test --directory=${directory}` + + `pattern pattern to search --pattern=${pattern}`, + ); +} -console.info('You can adjust the test with the following arguments:'); -console.info( - `directory target directory of test --directory=${TARGET_DIRECTORY}`, -); -console.info( - `pattern pattern to search --pattern=${PATTERN}`, -); -console.info(' '); -console.info('Start benchmark...'); -console.info(' '); +const fsWalkName = 'nodelib.fsWalk'; -suite.run({ - async: true, -}); +const suiteConfig = { + suiteName: 'crawl-file-system', + targetImplementation: fsWalkName, + cases: [ + // @TODO fix case execution + // FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory + /*[ + '@code-pushup/utils#crawlFileSystem', + callAndValidate( + crawlFileSystem, + { directory, pattern }, + '@code-pushup/utils#crawlFileSystem', + ), + ],*/ + [ + fsWalkName, + callAndValidate( + crawlFileSystemFsWalk, + { directory, pattern }, + fsWalkName, + ), + ], + ], +}; +export default suiteConfig; // ============================================================== -function wrapWithDefer( - asyncFn: (options: CrawlFileSystemOptions) => Promise, +const logged: Record = {}; +function callAndValidate>( + fn: (arg: T) => Promise, + options: T, + fnName: string, ) { - return { - defer: true, // important for async functions - fn: function (deferred: { resolve: () => void }) { - return asyncFn(options) - .catch(() => []) - .then((result: unknown[]) => { - if (result.length === 0) { - throw new Error(`Result length is ${result.length}`); - } else { - deferred.resolve(); - } - return void 0; - }); - }, + return async () => { + const result = await fn(options); + if (result.length === 0) { + throw new Error(`Result length is ${result.length}`); + } else { + if (!logged[fnName]) { + // eslint-disable-next-line functional/immutable-data + logged[fnName] = true; + // eslint-disable-next-line no-console + console.log( + `${chalk.bold(fnName)} found ${chalk.bold( + result.length, + )} files for pattern ${chalk.bold(options.pattern)}`, + ); + } + } }; } diff --git a/packages/utils/perf/glob-matching/index.ts b/packages/utils/perf/glob-matching/index.ts new file mode 100644 index 000000000..d205e42ff --- /dev/null +++ b/packages/utils/perf/glob-matching/index.ts @@ -0,0 +1,79 @@ +// eslint-ignore-next-line import/no-named-as-default-member +import fastGlob from 'fast-glob'; +import { glob } from 'glob'; +import { globby } from 'globby'; +import { join } from 'node:path'; +import yargs from 'yargs'; + +const cli = yargs(process.argv).options({ + pattern: { + type: 'array', + string: true, + default: [ + join(process.cwd(), '(packages|e2e|examples|testing|tools)/**/*.md'), + ], + }, + outputDir: { + type: 'string', + }, + logs: { + type: 'boolean', + default: true, + }, +}); + +// eslint-disable-next-line n/no-sync +const { pattern, logs } = cli.parseSync(); + +if (logs) { + // eslint-disable-next-line no-console + console.log('You can adjust the test with the following arguments:'); + // eslint-disable-next-line no-console + console.log( + `pattern glob pattern of test --pattern=${pattern.toString()}`, + ); +} + +const fastGlobName = 'fast-glob'; +const globName = 'glob'; +const globbyName = 'globby'; + +// ================== +const suiteConfig = { + suiteName: 'glob-matching', + targetImplementation: 'fast-glob', + cases: [ + // eslint-disable-next-line import/no-named-as-default-member + [fastGlobName, callAndValidate(fastGlob.async, pattern, fastGlobName)], + [globName, callAndValidate(glob, pattern, globName)], + [globbyName, callAndValidate(globby, pattern, globbyName)], + ], + time: 20_000, +}; +export default suiteConfig; + +// ============================================================== +const logged: Record = {}; +function callAndValidate( + fn: (patterns: T) => Promise, + globPatterns: T, + fnName: string, +) { + return async () => { + const result = await fn(globPatterns); + if (result.length === 0) { + // throw new Error(`Result length is ${result.length}`); + } else { + if (!logged[fnName]) { + // eslint-disable-next-line functional/immutable-data + logged[fnName] = true; + // eslint-disable-next-line no-console + console.log( + `${fnName} found ${result.length} files for pattern ${pattern.join( + ', ', + )}`, + ); + } + } + }; +} diff --git a/packages/utils/perf/glob/fast-glob.ts b/packages/utils/perf/glob/fast-glob.ts deleted file mode 100644 index 37ab5224e..000000000 --- a/packages/utils/perf/glob/fast-glob.ts +++ /dev/null @@ -1,5 +0,0 @@ -import * as fg from 'fast-glob'; - -export function fastGlob(pattern: string[]): Promise { - return fg.async(pattern); -} diff --git a/packages/utils/perf/glob/glob.ts b/packages/utils/perf/glob/glob.ts deleted file mode 100644 index 8b59973ab..000000000 --- a/packages/utils/perf/glob/glob.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { glob as g } from 'glob'; - -export function glob(pattern: string[]): Promise { - return g(pattern); -} diff --git a/packages/utils/perf/glob/globby.ts b/packages/utils/perf/glob/globby.ts deleted file mode 100644 index e88181fe7..000000000 --- a/packages/utils/perf/glob/globby.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { globby as g } from 'globby'; - -export function globby(pattern: string[]): Promise { - return g(pattern); -} diff --git a/packages/utils/perf/glob/index.ts b/packages/utils/perf/glob/index.ts deleted file mode 100644 index fa8f410b9..000000000 --- a/packages/utils/perf/glob/index.ts +++ /dev/null @@ -1,91 +0,0 @@ -import * as Benchmark from 'benchmark'; -import { join } from 'node:path'; -import { fastGlob } from './fast-glob'; -import { glob } from './glob'; -import { globby } from './globby'; - -const suite = new Benchmark.Suite('report-scoring'); - -const BASE_PATH = join( - process.cwd(), - '..', - '..', - '..', - 'node_modules', - '**/*.js', -); - -// ================== - -const start = performance.now(); - -// Add listener -const listeners = { - cycle: function (event: Benchmark.Event) { - console.info(String(event.target)); - }, - complete: () => { - if (typeof suite.filter === 'function') { - console.info(' '); - console.info( - `Total Duration: ${((performance.now() - start) / 1000).toFixed( - 2, - )} sec`, - ); - console.info(`Fastest is ${String(suite.filter('fastest').map('name'))}`); - } - }, -}; - -// ================== - -// Add tests -const pattern = [BASE_PATH]; -suite.add('glob', wrapWithDefer(glob)); -suite.add('globby', wrapWithDefer(globby)); -suite.add('fastGlob', wrapWithDefer(fastGlob)); - -// ================== - -// Add Listener -Object.entries(listeners).forEach(([name, fn]) => { - suite.on(name, fn); -}); - -// ================== - -console.info('You can adjust the test with the following arguments:'); -console.info(`pattern glob pattern of test --pattern=${BASE_PATH}`); -console.info(' '); -console.info('Start benchmark...'); -console.info(' '); - -suite.run({ - async: true, -}); - -// ============================================================== - -function wrapWithDefer(asyncFn: (pattern: string[]) => Promise) { - const logged: Record = {}; - return { - defer: true, // important for async functions - fn: function (deferred: { resolve: () => void }) { - return asyncFn(pattern) - .catch(() => []) - .then((result: unknown[]) => { - if (result.length === 0) { - throw new Error(`Result length is ${result.length}`); - } else { - if (!logged[asyncFn.name]) { - // eslint-disable-next-line functional/immutable-data - logged[asyncFn.name] = true; - console.info(`${asyncFn.name} found ${result.length} files`); - } - deferred.resolve(); - } - return void 0; - }); - }, - }; -} diff --git a/packages/utils/perf/score-report/index.ts b/packages/utils/perf/score-report/index.ts index 2dc427093..f1c1d640a 100644 --- a/packages/utils/perf/score-report/index.ts +++ b/packages/utils/perf/score-report/index.ts @@ -1,201 +1,61 @@ -import * as Benchmark from 'benchmark'; -import { Report } from '@code-pushup/models'; +import yargs from 'yargs'; import { scoreReport } from '../../src/lib/reports/scoring'; import { scoreReportOptimized0 } from './optimized0'; import { scoreReportOptimized1 } from './optimized1'; import { scoreReportOptimized2 } from './optimized2'; import { scoreReportOptimized3 } from './optimized3'; +import { minimalReport } from './utils'; -type MinimalReportOptions = { - numAuditsP1?: number; - numAuditsP2?: number; - numGroupRefs2?: number; -}; - -const PROCESS_ARGUMENT_NUM_AUDITS_P1 = Number.parseInt( - process.argv - .find(arg => arg.startsWith('--numAudits1')) - ?.split('=') - .at(-1) ?? '0', - 10, -); -const PROCESS_ARGUMENT_NUM_AUDITS_P2 = Number.parseInt( - process.argv - .find(arg => arg.startsWith('--numAudits2')) - ?.split('=') - .at(-1) ?? '0', - 10, -); -const PROCESS_ARGUMENT_NUM_GROUPS_P2 = Number.parseInt( - process.argv - .find(arg => arg.startsWith('--numGroupRefs2')) - ?.split('=') - .at(-1) ?? '0', - 10, -); - -const suite = new Benchmark.Suite('report-scoring'); - -const AUDIT_PREFIX = 'a-'; -const GROUP_PREFIX = 'g:'; -const PLUGIN_PREFIX = 'p.'; -const SLUG_PLUGIN_P1 = PLUGIN_PREFIX + 1; -const AUDIT_P1_PREFIX = AUDIT_PREFIX + SLUG_PLUGIN_P1; -const SLUG_PLUGIN_P2 = PLUGIN_PREFIX + 2; -const AUDIT_P2_PREFIX = AUDIT_PREFIX + SLUG_PLUGIN_P2; -const GROUP_P2_PREFIX = GROUP_PREFIX + SLUG_PLUGIN_P2; -const NUM_AUDITS_P1 = PROCESS_ARGUMENT_NUM_AUDITS_P1 || 27; -const NUM_AUDITS_P2 = PROCESS_ARGUMENT_NUM_AUDITS_P2 || 18; -const NUM_GROUPS_P2 = PROCESS_ARGUMENT_NUM_GROUPS_P2 || NUM_AUDITS_P2 / 2; - -// ================== - -// Add listener -const listeners = { - cycle: function (event: Benchmark.Event) { - console.info(String(event.target)); +const cli = yargs(process.argv).options({ + numAudits1: { + type: 'number', + default: 27, }, - complete: () => { - if (typeof suite.filter === 'function') { - console.info(' '); - console.info(`Fastest is ${String(suite.filter('fastest').map('name'))}`); - } + numAudits2: { + type: 'number', + default: 18, }, -}; - -// ================== - -// Add tests -suite.add('scoreReport', scoreReport); -suite.add('scoreReportOptimized0', scoreMinimalReportOptimized0); -suite.add('scoreReportOptimized1', scoreMinimalReportOptimized1); -suite.add('scoreReportOptimized2', scoreMinimalReportOptimized2); -suite.add('scoreReportOptimized3', scoreMinimalReportOptimized3); - -// ================== - -// Add Listener -Object.entries(listeners).forEach(([name, fn]) => { - suite.on(name, fn); -}); - -// ================== - -console.info('You can adjust the number of runs with the following arguments:'); -console.info( - `numAudits1 Number of audits in plugin 1. --numAudits1=${NUM_AUDITS_P1}`, -); -console.info( - `numAudits2 Number of audits in plugin 2. --numAudits2=${NUM_AUDITS_P2}`, -); -console.info( - `numGroupRefs2 Number of groups refs in plugin 2. --numGroupRefs2=${NUM_GROUPS_P2}`, -); -console.info(' '); -console.info('Start benchmark...'); -console.info(' '); - -const start = performance.now(); - -suite.run({ - onComplete: () => { - console.info( - `Total Duration: ${((performance.now() - start) / 1000).toFixed(2)} sec`, - ); + numGroupRefs2: { + type: 'number', + default: 6, + }, + logs: { + type: 'boolean', + default: false, }, }); -// ============================================================== - -function scoreMinimalReportOptimized0() { - scoreReportOptimized0(minimalReport()); -} - -function scoreMinimalReportOptimized1() { - scoreReportOptimized1(minimalReport()); -} +const { numAudits1, numAudits2, numGroupRefs2, logs } = await cli.parseAsync(); -function scoreMinimalReportOptimized2() { - scoreReportOptimized2(minimalReport()); -} +// ================== -function scoreMinimalReportOptimized3() { - scoreReportOptimized3(minimalReport()); +if (logs) { + // eslint-disable-next-line no-console + console.log( + 'You can adjust the number of runs with the following arguments:' + + `numAudits1 Number of audits in plugin 1. --numAudits1=${numAudits1}` + + `numAudits2 Number of audits in plugin 2. --numAudits2=${numAudits2}` + + `numGroupRefs2 Number of groups refs in plugin 2. --numGroupRefs2=${numGroupRefs2}`, + ); } - // ============================================================== +const options = { numAudits1, numAudits2, numGroupRefs2 }; -// eslint-disable-next-line max-lines-per-function -function minimalReport(opt?: MinimalReportOptions): Report { - const numAuditsP1 = opt?.numAuditsP1 ?? NUM_AUDITS_P1; - const numAuditsP2 = opt?.numAuditsP2 ?? NUM_AUDITS_P2; - const numGroupRefs2 = opt?.numGroupRefs2 ?? NUM_GROUPS_P2; - - return { - date: '2022-01-01', - duration: 0, - categories: [ - { - slug: 'c1_', - title: 'Category 1', - refs: Array.from({ length: numAuditsP1 }).map((_, idx) => ({ - type: 'audit', - plugin: SLUG_PLUGIN_P1, - slug: `${AUDIT_P1_PREFIX}${idx}`, - weight: 1, - })), - isBinary: false, - }, - { - slug: 'c2_', - title: 'Category 2', - refs: Array.from({ length: numAuditsP2 }).map((_, idx) => ({ - type: 'audit', - plugin: SLUG_PLUGIN_P2, - slug: `${AUDIT_P2_PREFIX}${idx}`, - weight: 1, - })), - isBinary: false, - }, - ], - plugins: [ - { - date: '2022-01-01', - duration: 0, - slug: SLUG_PLUGIN_P1, - title: 'Plugin 1', - icon: 'slug', - audits: Array.from({ length: numAuditsP1 }).map((_, idx) => ({ - value: 0, - slug: `${AUDIT_P1_PREFIX}${idx}`, - title: 'Default Title', - score: 0.1, - })), - groups: [], - }, - { - date: '2022-01-01', - duration: 0, - slug: SLUG_PLUGIN_P2, - title: 'Plugin 2', - icon: 'slug', - audits: Array.from({ length: numAuditsP2 }).map((_, idx) => ({ - value: 0, - slug: `${AUDIT_P2_PREFIX}${idx}`, - title: 'Default Title', - score: 0.1, - })), - groups: [ - { - title: 'Group 1', - slug: GROUP_P2_PREFIX + 1, - refs: Array.from({ length: numGroupRefs2 }).map((_, idx) => ({ - slug: `${AUDIT_P2_PREFIX}${idx}`, - weight: 1, - })), - }, - ], - }, +// Add tests +const suiteConfig = { + suiteName: 'score-report', + targetImplementation: '@code-pushup/utils#scoreReport', + cases: [ + [ + '@code-pushup/utils#scoreReport', + () => scoreReport(minimalReport(options)), ], - }; -} + ['scoreReportv0', () => scoreReportOptimized0(minimalReport(options))], + ['scoreReportv1', () => scoreReportOptimized1(minimalReport(options))], + ['scoreReportv2', () => scoreReportOptimized2(minimalReport(options))], + ['scoreReportv3', () => scoreReportOptimized3(minimalReport(options))], + ], + time: 2000, +}; + +export default suiteConfig; diff --git a/packages/utils/perf/score-report/utils.ts b/packages/utils/perf/score-report/utils.ts new file mode 100644 index 000000000..3ce529fce --- /dev/null +++ b/packages/utils/perf/score-report/utils.ts @@ -0,0 +1,99 @@ +import { AuditReport, GroupRef, Report } from '@code-pushup/models'; + +const AUDIT_PREFIX = 'a-'; +const GROUP_PREFIX = 'g:'; +const PLUGIN_PREFIX = 'p.'; +const SLUG_PLUGIN_P1 = PLUGIN_PREFIX + 1; +const AUDIT_P1_PREFIX = AUDIT_PREFIX + SLUG_PLUGIN_P1; +const SLUG_PLUGIN_P2 = PLUGIN_PREFIX + 2; +const AUDIT_P2_PREFIX = AUDIT_PREFIX + SLUG_PLUGIN_P2; +const GROUP_P2_PREFIX = GROUP_PREFIX + SLUG_PLUGIN_P2; + +type MinimalReportOptions = { + numAudits1?: number; + numAudits2?: number; + numGroupRefs2?: number; +}; + +// eslint-disable-next-line max-lines-per-function +export function minimalReport(cfg: MinimalReportOptions = {}): Report { + return { + date: '2024-12-11', + duration: 0, + packageName: '@code-pushup/cli', + version: '1.0.0', + categories: [ + { + slug: 'c1_', + title: 'Category 1', + refs: Array.from({ length: cfg.numAudits1 }).map((_, idx) => ({ + type: 'audit', + plugin: SLUG_PLUGIN_P1, + slug: `${AUDIT_P1_PREFIX}${idx}`, + weight: 1, + })), + isBinary: false, + }, + { + slug: 'c2_', + title: 'Category 2', + refs: Array.from({ length: cfg.numAudits2 }).map((_, idx) => ({ + type: 'audit', + plugin: SLUG_PLUGIN_P2, + slug: `${AUDIT_P2_PREFIX}${idx}`, + weight: 1, + })), + isBinary: false, + }, + ], + plugins: [ + { + date: '2022-01-01', + duration: 0, + slug: SLUG_PLUGIN_P1, + title: 'Plugin 1', + icon: 'slug', + audits: Array.from({ length: cfg.numAudits1 }).map( + (_, idx) => + ({ + slug: `${AUDIT_P1_PREFIX}${idx}`, + title: 'Default Title', + score: 0.1, + value: 0, + displayValue: '0', + } satisfies AuditReport), + ), + groups: [], + }, + { + date: '2022-01-01', + duration: 0, + slug: SLUG_PLUGIN_P2, + title: 'Plugin 2', + icon: 'slug', + audits: Array.from({ length: cfg.numAudits2 }).map( + (_, idx) => + ({ + value: 0, + slug: `${AUDIT_P2_PREFIX}${idx}`, + title: 'Default Title', + score: 0.1, + } satisfies AuditReport), + ), + groups: [ + { + title: 'Group 1', + slug: GROUP_P2_PREFIX + 1, + refs: Array.from({ length: cfg.numGroupRefs2 }).map( + (_, idx) => + ({ + slug: `${AUDIT_P2_PREFIX}${idx}`, + weight: 1, + } satisfies GroupRef), + ), + }, + ], + }, + ], + } satisfies Report; +} diff --git a/packages/utils/project.json b/packages/utils/project.json index 9cc679f36..9be7d1f6f 100644 --- a/packages/utils/project.json +++ b/packages/utils/project.json @@ -30,16 +30,7 @@ } }, "perf": { - "command": "npx tsx --tsconfig=../tsconfig.perf.json", - "options": { - "cwd": "./packages/utils/perf" - } - }, - "perf:list": { - "command": "ls -l | grep \"^d\" | awk '{print $9}'", - "options": { - "cwd": "./packages/utils/perf" - } + "command": "node tools/benchmark/bin.mjs" }, "unit-test": { "executor": "@nx/vite:test", diff --git a/tools/benchmark/benchmark.runner.mjs b/tools/benchmark/benchmark.runner.mjs new file mode 100644 index 000000000..fd760f479 --- /dev/null +++ b/tools/benchmark/benchmark.runner.mjs @@ -0,0 +1,41 @@ +import Benchmark from 'benchmark'; + +export default { + run: async ( + { suiteName, cases, targetImplementation, tsconfig }, + options = { verbose: false }, + ) => { + const { verbose, maxTime } = options; + + return new Promise((resolve, reject) => { + const suite = new Benchmark.Suite(suiteName); + + // Add Listener + Object.entries({ + error: e => reject(e?.target?.error ?? e), + cycle: function (event) { + verbose && console.log(String(event.target)); + }, + complete: event => { + const fastest = String(suite.filter('fastest').map('name')[0]); + const json = event.currentTarget.map(bench => ({ + suiteName, + name: bench.name || '', + hz: bench.hz ?? 0, // operations per second + rme: bench.stats?.rme ?? 0, // relative margin of error + samples: bench.stats?.sample.length ?? 0, // number of samples + isFastest: fastest === bench.name, + isTarget: targetImplementation === bench.name, + })); + + resolve(json); + }, + }).forEach(([name, fn]) => suite.on(name, fn)); + + // register test cases + cases.forEach(tuple => suite.add(...tuple)); + + suite.run({ async: true }); + }); + }, +}; diff --git a/tools/benchmark/benny.runner.mjs b/tools/benchmark/benny.runner.mjs new file mode 100644 index 000000000..c368c45c0 --- /dev/null +++ b/tools/benchmark/benny.runner.mjs @@ -0,0 +1,57 @@ +import benny from 'benny'; +import { join } from 'node:path'; +import { ensureDirectoryExists } from '../../packages/utils/src'; + +export const bennyRunner = { + run: async ({ suiteName, cases, targetImplementation }, options = {}) => { + const { outputFolder = join('js-benchmarking') } = options; + + return new Promise(resolve => { + // This is not working with named imports + // eslint-disable-next-line import/no-named-as-default-member + const suite = benny.suite( + suiteName, + + ...cases.map(([name, fn]) => + benny.add(name, () => { + fn(); + }), + ), + + benny.cycle(), + + benny.complete(summary => { + console.log(summary); + resolve( + benchToBenchmarkResult(summary, { + suiteName, + cases, + targetImplementation, + }), + ); + }), + benny.save({ file: 'reduce', version: '1.0.0' }), + benny.save({ file: 'reduce', format: 'json', folder: outputFolder }), + ); + }); + }, +}; + +export function benchToBenchmarkResult( + suite, + { targetImplementation, suiteName }, +) { + const { name } = suite.fastest; + return suite.results.map(bench => ({ + suiteName, + name: bench.name || '', + hz: bench.ops ?? 0, // operations per second + rme: bench.margin ?? 0, // relative margin of error + // samples not given by benny + samples: 0, // samples recorded for this case + isFastest: name === bench.name, + isTarget: targetImplementation === bench.name, + })); +} + +export default bennyRunner; diff --git a/tools/benchmark/bin.mjs b/tools/benchmark/bin.mjs new file mode 100644 index 000000000..4ea6e3b7d --- /dev/null +++ b/tools/benchmark/bin.mjs @@ -0,0 +1,122 @@ +import { writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import yargs from 'yargs'; +import benchmarkRunner from './benchmark.runner.mjs'; +import bennyRunner from './benny.runner.mjs'; +import tinybenchRunner from './tinybench.runner.mjs'; +import { loadSuits } from './utils.mjs'; + +const supportedRunner = new Set(['tinybench', 'benchmark', 'benny']); +const cli = yargs(process.argv).options({ + targets: { + type: 'array', + default: [], + }, + tsconfig: { + type: 'string', + }, + runner: { + type: 'string', + default: 'tinybench', + }, + outputDir: { + type: 'string', + default: 'tmp', + }, + verbose: { + type: 'boolean', + default: true, + }, +}); + +(async () => { + let { + targets = [], + verbose, + tsconfig, + outputDir, + runner, + } = await cli.parseAsync(); + + if (!supportedRunner.has(runner)) { + runner = 'tinybench'; + } + + if (targets.length === 0) { + throw Error('No targets given. Use `--targets=suite1.ts` to set targets.'); + } + + // execute benchmark + const allSuits = await loadSuits(targets, { tsconfig }); + if (verbose) { + console.log( + `Loaded targets: ${allSuits + .map(({ suiteName }) => suiteName) + .join(', ')}`, + ); + } + // create audit output + const allSuiteResults = []; + // Execute each suite sequentially + for (const suite of allSuits) { + let runnerFn; + switch (runner) { + case 'tinybench': + runnerFn = tinybenchRunner; + break; + case 'benchmark': + runnerFn = benchmarkRunner; + break; + case 'benny': + runnerFn = bennyRunner; + break; + default: + runnerFn = tinybenchRunner; + } + const result = await runnerFn.run(suite); + allSuiteResults.push(result); + } + console.log(`Use ${runner} for benchmarking`); + + allSuiteResults.forEach(async results => { + const { + suiteName, + name, + hz: maxHz, + } = results.find(({ isFastest }) => isFastest); + const target = results.find(({ isTarget }) => isTarget); + console.log( + `In suite ${suiteName} fastest is: ${name} target is ${target?.name}`, + ); + if (outputDir) { + await writeFile( + join(outputDir, `${suiteName}-${runner}-${Date.now()}.json`), + JSON.stringify( + results.map(({ name, hz, rme, samples }) => ({ + name, + hz, + rme, + samples, + })), + null, + 2, + ), + ); + } + console.table( + results.map(({ name, hz, rme, samples, isTarget, isFastest }) => { + const targetIcon = isTarget ? '🎯' : ''; + const postfix = isFastest + ? '(fastest 🔥)' + : `(${((1 - hz / maxHz) * 100).toFixed(1)}% slower)`; + return { + // fast-glob x 40,824 ops/sec ±4.44% (85 runs sampled) + message: `${targetIcon}${name} x ${hz.toFixed( + 2, + )} ops/sec ±${rme.toFixed(2)}; ${samples} samples ${postfix}`, + severity: hz < maxHz && isTarget ? 'error' : 'info', + }; + }), + ); + }); +})(); diff --git a/tools/benchmark/docs/README.md b/tools/benchmark/docs/README.md new file mode 100644 index 000000000..ce01f3c12 --- /dev/null +++ b/tools/benchmark/docs/README.md @@ -0,0 +1,26 @@ +# Micro Benchmarks + +## CLI + +Execute any benchmark by running `node tools/benchmark/bin.mjs --targets ` + +**Result:** +![benchmark-terminal-result.png](benchmark-terminal-result.png) + +## Helper + +```ts +import { type SuiteConfig, runSuit } from './benchmark/suite-helper.ts'; + +const suite: SuiteConfig = { + suiteName: 'dummy-suite', + targetImplementation: 'version-2', + cases: [ + ['version-1', async () => new Promise(resolve => setTimeout(resolve, 30))], + ['version-2', async () => new Promise(resolve => setTimeout(resolve, 50))], + ['version-3', async () => new Promise(resolve => setTimeout(resolve, 80))], + ], +}; +const results = await runSuite(suite); +console.log(`Fastest is: ${results.filter(({ isFastest }) => isFastest)}`); +``` diff --git a/tools/benchmark/docs/benchmark-terminal-result.png b/tools/benchmark/docs/benchmark-terminal-result.png new file mode 100644 index 000000000..31fe937cb Binary files /dev/null and b/tools/benchmark/docs/benchmark-terminal-result.png differ diff --git a/tools/benchmark/tinybench.runner.mjs b/tools/benchmark/tinybench.runner.mjs new file mode 100644 index 000000000..4dc1d4f56 --- /dev/null +++ b/tools/benchmark/tinybench.runner.mjs @@ -0,0 +1,57 @@ +import { Bench } from 'tinybench'; + +export default { + run: async ({ suiteName, cases, targetImplementation, time = 1000 }) => { + // This is not working with named imports + // eslint-disable-next-line import/no-named-as-default-member + const suite = new Bench({ time }); + + // register test cases + cases.forEach(tuple => suite.add(...tuple)); + + await suite.warmup(); // make results more reliable, ref: https://github.com/tinylibs/tinybench/pull/50 + await suite.run(); + + return benchToBenchmarkResult(suite, { + suiteName, + cases, + targetImplementation, + }); + }, +}; + +export function benchToBenchmarkResult(bench, suite) { + const caseNames = suite.cases.map(([name]) => name); + const results = caseNames.map(caseName => { + const result = bench.getTask(caseName)?.result ?? {}; + return { + suiteName: suite.suiteName, + name: caseName, + hz: result.hz, + rme: result.rme, + samples: result.samples.length, + isTarget: suite.targetImplementation === caseName, + isFastest: false, // preliminary result + }; + }); + + const fastestName = + caseNames.reduce( + (fastest, name) => { + const { hz } = bench.getTask(name)?.result ?? {}; + if (fastest.name === undefined) { + return { hz, name }; + } + if (hz && fastest.hz && hz > fastest.hz) { + return { hz, name }; + } + return fastest; + }, + { hz: 0, name: undefined }, + ).name ?? ''; + + return results.map(result => ({ + ...result, + isFastest: fastestName === result.name, + })); +} diff --git a/tools/benchmark/utils.mjs b/tools/benchmark/utils.mjs new file mode 100644 index 000000000..1cc6b2283 --- /dev/null +++ b/tools/benchmark/utils.mjs @@ -0,0 +1,74 @@ +import Benchmark from 'benchmark'; +import { bundleRequire } from 'bundle-require'; + +export class NoExportError extends Error { + constructor(filepath) { + super(`No default export found in ${filepath}`); + } +} + +export async function importEsmModule(options) { + const { mod } = await bundleRequire({ + format: 'esm', + ...options, + }); + + if (!('default' in mod)) { + throw new NoExportError(options.filepath); + } + return mod.default; +} + +export function loadSuits(targets, options) { + const { tsconfig } = options; + return Promise.all( + targets.map(suitePath => + importEsmModule( + tsconfig + ? { + tsconfig, + filepath: suitePath, + } + : { filepath: suitePath }, + ), + ), + ); +} + +export function runSuite( + { suiteName, cases, targetImplementation, tsconfig }, + options = { verbose: false }, +) { + const { verbose, maxTime } = options; + + return new Promise((resolve, reject) => { + const suite = new Benchmark.Suite(suiteName); + + // Add Listener + Object.entries({ + error: e => reject(e?.target?.error ?? e), + cycle: function (event) { + verbose && console.log(String(event.target)); + }, + complete: event => { + const fastest = String(suite.filter('fastest').map('name')[0]); + const json = event.currentTarget.map(bench => ({ + suiteName, + name: bench.name || '', + hz: bench.hz ?? 0, // operations per second + rme: bench.stats?.rme ?? 0, // relative margin of error + samples: bench.stats?.sample.length ?? 0, // number of samples + isFastest: fastest === bench.name, + isTarget: targetImplementation === bench.name, + })); + + resolve(json); + }, + }).forEach(([name, fn]) => suite.on(name, fn)); + + // register test cases + cases.forEach(tuple => suite.add(...tuple)); + + suite.run({ async: true }); + }); +}