refactor multiple benchmark support

Add benchmarks via a generic register function.
Each benchmark is implemented by a single function.
Each benchmark has its own tests defined.

Author: hoeck

graph.ts benchmarks/helpers/graph.tsgraph.ts renamed to benchmarks/helpers/graph.ts

@@ -0,0 +1,131 @@
+import { writeFile, readdirSync, readFileSync, writeFileSync } from 'fs';
+import { join } from 'path';
+import { suite, add, cycle, complete, save } from 'benny';
+import stringify from 'csv-stringify/lib/sync';
+import pkg from '../../package.json';
+import { graph } from './graph';
+import { BenchmarkCase } from './types';
+import { getRegisteredBenchmarks, availableBenchmarks } from './register';
+
+const RESULTS_DIR = join(__dirname, '../../results');
+const DOCS_DIR = join(__dirname, '../../docs');
+const NODE_VERSION = process.env.NODE_VERSION || process.version;
+
+export async function main() {
+  for (let [benchmarkKey, benchmarks] of getRegisteredBenchmarks()) {
+    await runBenchmarks(benchmarkKey, benchmarks);
+  }
+
+  aggregateResults();
+}
+
+async function runBenchmarks(name: string, cases: BenchmarkCase[]) {
+  const fns = cases.map(c => add(c.moduleName, () => c.run()));
+
+  return suite(
+    name,
+
+    // benchmark functions
+    ...fns,
+
+    cycle(),
+    complete(),
+
+    // saves CSV and SVG
+    complete(async res => {
+      const csvArr = res.results.map(({ name, ops }) => {
+        return { name, ops };
+      });
+
+      const csv = stringify(csvArr, {
+        header: true,
+        columns: ['name', 'ops'],
+      });
+      const csvFilename = join(RESULTS_DIR, getResultFileName(name, '.csv'));
+      await saveFile(csvFilename, csv);
+
+      const svg = await graph(csvFilename);
+      const svgFilename = join(RESULTS_DIR, getResultFileName(name, '.svg'));
+      await saveFile(svgFilename, svg);
+    }),
+
+    // saves raw JSON results
+    save({
+      file: getResultFileName(name, ''),
+      folder: RESULTS_DIR,
+      version: pkg.version,
+    })
+  );
+}
+
+function saveFile(filename: string, content: string) {
+  return new Promise<void>((resolve, reject) => {
+    return writeFile(filename, content, { encoding: 'utf8' }, err =>
+      err ? reject(err) : resolve()
+    );
+  });
+}
+
+function getResultFileName(
+  benchmarkName: string,
+  suffix: '.json' | '.svg' | '.csv' | ''
+) {
+  return `${benchmarkName}-${NODE_VERSION}${suffix}`;
+}
+
+function getNodeMajorVersion() {
+  let majorVersion: number = 0;
+
+  majorVersion = parseInt(NODE_VERSION);
+
+  if (!isNaN(majorVersion)) {
+    return majorVersion;
+  }
+
+  majorVersion = parseInt(NODE_VERSION.slice(1));
+
+  if (!isNaN(majorVersion)) {
+    return majorVersion;
+  }
+
+  return majorVersion;
+}
+
+// combine all benchmarks into a single file for this node version
+function aggregateResults() {
+  const majorVersion = getNodeMajorVersion();
+
+  const results: {
+    name: string;
+    benchmark: string;
+    nodeVersion: string;
+    ops: number;
+    margin: number;
+  }[] = [];
+
+  Object.keys(availableBenchmarks).forEach(name => {
+    const data = JSON.parse(
+      readFileSync(
+        join(RESULTS_DIR, getResultFileName(name, '.json'))
+      ).toString()
+    );
+
+    data.results.forEach((r: any) =>
+      results.push({
+        name: r.name,
+        benchmark: name,
+        nodeVersion: NODE_VERSION,
+        ops: r.ops,
+        margin: r.margin,
+      })
+    );
+  });
+
+  writeFileSync(
+    join(DOCS_DIR, 'results', `node-${majorVersion}.json`),
+
+    JSON.stringify({
+      results,
+    })
+  );
+}

benchmarks/helpers/main.ts

@@ -0,0 +1,48 @@
+import { BenchmarkCase } from './types';
+import { Validate } from '../validate';
+import { ValidateStrict } from '../validateStrict';
+
+export const availableBenchmarks = {
+  validate: Validate,
+  validateStrict: ValidateStrict,
+};
+type AvailableBenchmarks = typeof availableBenchmarks;
+export type AvailableBenchmarksIds = keyof AvailableBenchmarks;
+
+const registeredBenchmarks = new Map<AvailableBenchmarksIds, BenchmarkCase[]>();
+
+export function getRegisteredBenchmarks(): [
+  keyof AvailableBenchmarks,
+  BenchmarkCase[]
+][] {
+  return [...registeredBenchmarks.entries()];
+}
+
+/**
+ * Add a benchmark implementation.
+ */
+export function register<
+  K extends keyof AvailableBenchmarks,
+  I = AvailableBenchmarks[K]['prototype']['fn']
+>(moduleName: string, benchmarkId: K, implementation: I) {
+  let benchmarks = registeredBenchmarks.get(benchmarkId);
+
+  if (!benchmarks) {
+    benchmarks = [];
+
+    registeredBenchmarks.set(benchmarkId, benchmarks);
+  }
+
+  if (benchmarks.find(c => c.moduleName === moduleName)) {
+    console.error(
+      'benchmark',
+      benchmarkId,
+      'is already defined for module',
+      moduleName
+    );
+  }
+
+  const benchmarkCtor = availableBenchmarks[benchmarkId];
+
+  benchmarks.push(new benchmarkCtor(moduleName, implementation as any));
+}

benchmarks/helpers/register.ts

@@ -0,0 +1,28 @@
+export interface BenchmarkCase {
+  readonly moduleName: string;
+
+  // execute the actual benchmark function
+  run(): void;
+
+  // run the benchmarks jest test
+  test(): void;
+}
+
+export abstract class Benchmark<Fn> implements BenchmarkCase {
+  // name of the module that is benchmarked
+  readonly moduleName: string;
+
+  // the function that implements the benchmark
+  readonly fn: Fn;
+
+  constructor(moduleName: string, fn: Fn) {
+    this.moduleName = moduleName;
+    this.fn = fn;
+  }
+
+  // execute the actual benchmark function
+  abstract run(): void;
+
+  // run the benchmarks jest test
+  abstract test(): void;
+}

benchmarks/helpers/types.ts

@@ -0,0 +1,6 @@
+export {
+  AvailableBenchmarksIds,
+  getRegisteredBenchmarks,
+  register,
+} from './helpers/register';
+export { main } from './helpers/main';

benchmarks/index.ts

@@ -0,0 +1,41 @@
+import { Benchmark } from './helpers/types';
+
+const validateData = Object.freeze({
+  number: 1,
+  negNumber: -1,
+  maxNumber: Number.MAX_VALUE,
+  string: 'string',
+  longString:
+    'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Vivendum intellegat et qui, ei denique consequuntur vix. Semper aeterno percipit ut his, sea ex utinam referrentur repudiandae. No epicuri hendrerit consetetur sit, sit dicta adipiscing ex, in facete detracto deterruisset duo. Quot populo ad qui. Sit fugit nostrum et. Ad per diam dicant interesset, lorem iusto sensibus ut sed. No dicam aperiam vis. Pri posse graeco definitiones cu, id eam populo quaestio adipiscing, usu quod malorum te. Ex nam agam veri, dicunt efficiantur ad qui, ad legere adversarium sit. Commune platonem mel id, brute adipiscing duo an. Vivendum intellegat et qui, ei denique consequuntur vix. Offendit eleifend moderatius ex vix, quem odio mazim et qui, purto expetendis cotidieque quo cu, veri persius vituperata ei nec. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.',
+  boolean: true,
+  deeplyNested: {
+    foo: 'bar',
+    num: 1,
+    bool: false,
+  },
+});
+
+type Fn = (data: typeof validateData) => void;
+
+export class Validate extends Benchmark<Fn> {
+  run() {
+    this.fn(validateData);
+  }
+
+  test() {
+    describe(this.moduleName, () => {
+      test(`should validate the data`, () => {
+        expect(this.fn(validateData)).toEqual(validateData);
+      });
+
+      test(`should err on invalid data`, () => {
+        expect(() =>
+          this.fn({
+            ...validateData,
+            number: 'foo',
+          } as any)
+        ).toThrow();
+      });
+    });
+  }
+}

benchmarks/validate.ts

@@ -0,0 +1,60 @@
+import { Benchmark } from './helpers/types';
+
+const validateData = Object.freeze({
+  number: 1,
+  negNumber: -1,
+  maxNumber: Number.MAX_VALUE,
+  string: 'string',
+  longString:
+    'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Vivendum intellegat et qui, ei denique consequuntur vix. Semper aeterno percipit ut his, sea ex utinam referrentur repudiandae. No epicuri hendrerit consetetur sit, sit dicta adipiscing ex, in facete detracto deterruisset duo. Quot populo ad qui. Sit fugit nostrum et. Ad per diam dicant interesset, lorem iusto sensibus ut sed. No dicam aperiam vis. Pri posse graeco definitiones cu, id eam populo quaestio adipiscing, usu quod malorum te. Ex nam agam veri, dicunt efficiantur ad qui, ad legere adversarium sit. Commune platonem mel id, brute adipiscing duo an. Vivendum intellegat et qui, ei denique consequuntur vix. Offendit eleifend moderatius ex vix, quem odio mazim et qui, purto expetendis cotidieque quo cu, veri persius vituperata ei nec. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur.',
+  boolean: true,
+  deeplyNested: {
+    foo: 'bar',
+    num: 1,
+    bool: false,
+  },
+});
+
+type Fn = (data: typeof validateData) => void;
+
+export class ValidateStrict extends Benchmark<Fn> {
+  run() {
+    this.fn(validateData);
+  }
+
+  test() {
+    describe(this.moduleName, () => {
+      test(`should validate the data`, () => {
+        expect(this.fn(validateData)).toEqual(validateData);
+      });
+
+      test(`should throw on invalid attribute type`, () => {
+        expect(() =>
+          this.fn({
+            ...validateData,
+            number: 'foo',
+          } as any)
+        ).toThrow();
+      });
+
+      test(`should throw on extra attributes`, () => {
+        expect(() =>
+          this.fn({
+            ...validateData,
+            extraAttribute: true,
+          } as any)
+        ).toThrow();
+      });
+
+      test(`should throw on missing attributes`, () => {
+        const data: any = {
+          ...validateData,
+        };
+
+        delete data.number;
+
+        expect(() => this.fn(data)).toThrow();
+      });
+    });
+  }
+}

benchmarks/validateStrict.ts

@@ -14,42 +14,18 @@
 // import { PurifyCase } from './purify-ts';
 // import { RulrCase } from './rulr';
 // import { RuntypesCase } from './runtypes';
-import { SimpleRuntypesCase } from './simple-runtypes';
+// import { SimpleRuntypesCase } from './simple-runtypes';
 // import { SuperstructCase } from './superstruct';
 // import { SuretypeCase } from './suretype';
 // import { ToiCase } from './toi';
-import { TsInterfaceCheckerCase } from './ts-interface-checker';
+// import { TsInterfaceCheckerCase } from './ts-interface-checker';
 // import { TsJsonValidatorCase } from './ts-json-validator';
 // import { TsUtilsCase } from './ts-utils';
 // import { TypeOfWebSchemaCase } from './typeofweb-schema';
-import { ValitaCase } from './valita';
+// import { ValitaCase } from './valita';
 // import { YupCase } from './yup';
 // import { ZodCase } from './zod';
 
-export const cases = [
-  // BuenoCase,
-  // ClassValidatorAsyncCase,
-  // ClassValidatorSyncCase,
-  // ComputedTypesCase,
-  // DecodersCase,
-  // IoTsCase,
-  // JointzCase,
-  // JsonDecoderCase,
-  // MarshalCase,
-  // MojoTechJsonTypeValidationCase,
-  // MyzodCase,
-  // PurifyCase,
-  // RulrCase,
-  // RuntypesCase,
-  SimpleRuntypesCase,
-  // SuperstructCase,
-  // SuretypeCase,
-  // ToiCase,
-  TsInterfaceCheckerCase,
-  // TsJsonValidatorCase,
-  // TsUtilsCase,
-  // TypeOfWebSchemaCase,
-  ValitaCase,
-  // YupCase,
-  // ZodCase,
-];
+import './simple-runtypes';
+import './valita';
+import './runtypes';

cases/index.ts

@@ -1,5 +1,5 @@
 import { Boolean, Number, String, Record } from 'runtypes';
-import { Case } from './abstract';
+import { register } from '../benchmarks';
 
 const dataType = Record({
   number: Number,
@@ -15,10 +15,6 @@ const dataType = Record({
   }),
 });
 
-export class RuntypesCase extends Case implements Case {
-  name = 'runtypes';
-
-  validate() {
-    return dataType.check(this.data);
-  }
-}
+register('runtypes', 'validate', data => {
+  return dataType.check(data);
+});