Restructure benchmarks and scenarios

README.md

@@ -25,4 +25,4 @@ The original `state` is not modified.
 - If you're using [lodash](https://lodash.com) then you don't need this: [see this thread for alternatives](https://github.com/lodash/lodash/issues/1696#issuecomment-328335502).
 
 - The small bundle size of this library comes with a slight speed tradeoff. This library is a little slower than
-  immutable-assign and immutable.js, but a little faster than Immer and seamless-immutable.
+  immutable-assign and immutable.js, but a little faster than Immer and seamless-immutable. [See full benchmark here.](./benchmark)

benchmark/README.md

@@ -3,7 +3,7 @@
 This project exercises several different utilities for deep-setting immutable data.
 
 Note that these numbers ONLY reflect `set` operations. Many of these libraries also offer `get` functionality, so
-bundle sizes are not directly comparable. These are for reference only.
+the final bundle size will depend on your use cases. These are for reference only.
 
 ## Usage
 
@@ -26,3 +26,19 @@ Last run on (date goes here)
 | [lodash](https://lodash.com/) (setWith + clone)                                 |     `4.17.20` | [![gzip size](https://img.shields.io/bundlephobia/minzip/lodash)](https://bundlephobia.com/result?p=lodash)                         |              (pending) |
 | [seamless-immutable](https://github.com/rtfeldman/seamless-immutable)           |       `7.1.4` | [![gzip size](https://img.shields.io/bundlephobia/minzip/seamless-immutable)](https://bundlephobia.com/result?p=seamless-immutable) |              (pending) |
 | [tiny-immutable-set](https://github.com/spautz/tiny-immutable-set)              |       `0.1.0` | [![gzip size](https://img.shields.io/bundlephobia/minzip/tiny-immutable-set)](https://bundlephobia.com/result?p=tiny-immutable-set) |              (pending) |
+
+## Organization
+
+Each library has a file under `libraries/` which provides a callback for each scenario:
+
+- `setWithString(object, path, value)`, where path looks like `"prop1.prop2.prop3"`
+- `setWithArray(object, path, value)`, where path looks like `["prop1", "prop2", "prop3"]`
+- `setWithArrayString(object, path, value)`, where path looks like `"prop1[prop2][prop3]"`
+
+Each scenario type has a file under `scenarios/` which provides setup and teardown callbacks:
+
+- `setup(numCasesToGenerate)` returns an array of `(object, path, value)` arguments to use for each test case
+- `teardown(cases)` is available if you need it
+
+`run-benchmark.js` runs everything based on the libraries and scenarios exported from each directory's `index.js`,
+with some additional settings in `options.js`.

benchmark/libraries/immer.js

@@ -5,7 +5,7 @@ const setWithArray = (obj, path, value) => {
     let index = 0;
     const pathToTraverse = path.length - 1;
     for (; index < pathToTraverse; index++) {
-      draftObj = draftObj[path[index]];
+      draftObj = draftObj[path[index]] || {};
     }
     draftObj[path[index]] = value;
   });

benchmark/libraries/index.js

@@ -0,0 +1,8 @@
+module.exports = [
+  require('./immer'),
+  require('./immutable'),
+  require('./immutable-assign'),
+  require('./lodash'),
+  require('./seamless-immutable'),
+  require('./tiny-immutable-set'),
+];

benchmark/options.js

@@ -0,0 +1,16 @@
+const options = {
+  /* How many times each scenario is run for each library */
+  numIterations: 10000,
+  /* Whether to console.log while libraries and scenarios are running */
+  showLogs: true,
+  /* Whether to console.warn if a library hasn't implemented a scenario */
+  showWarnings: true,
+
+  // To check against order-of-execution issues, we can run libraries-for-each-scenario, scenarios-for-each-library,
+  // or both. This is mostly to validate the benchmark runner itself. (Running both will double the number of
+  // iterations, and it won't give you any more information since execution order doesn't matter.)
+  loopLibrariesThenScenarios: true,
+  loopScenariosThenLibraries: true,
+};
+
+module.exports = options;

benchmark/run-benchmark.js

@@ -1,38 +1,11 @@
 const { Timeline } = require('@nelsongomes/ts-timeframe');
 
-const {
-  generateScenarios,
-  runStringScenario,
-  runArrayScenario,
-  runArrayStringScenario,
-} = require('./scenarios');
-
-const allLibraries = [
-  require('./libraries/immer'),
-  require('./libraries/immutable'),
-  require('./libraries/immutable-assign'),
-  require('./libraries/lodash'),
-  require('./libraries/seamless-immutable'),
-  require('./libraries/tiny-immutable-set'),
-];
-
-const allScenarios = generateScenarios({
-  scenarioList: [
-    // runStringScenario,
-    // runArrayScenario,
-    // runArrayStringScenario,
-    // // Weight the plain scenarios more, since those are likely the most common case for most people
-    // runStringScenario,
-    // runArrayScenario,
-    runArrayStringScenario,
-    runArrayScenario,
-    runArrayStringScenario,
-    runArrayScenario,
-  ],
-  iterationsPerRun: 50000,
-});
+const options = require('./options');
+const allLibraries = require('./libraries');
+const allScenarios = require('./scenarios');
 
 function accumulateTime(timeInfo, timeLineEvent) {
+  timeInfo = timeInfo || { seconds: 0, nanoseconds: 0, eventCount: 0 };
   const [seconds, nanoseconds] = timeLineEvent.getDurationRaw();
 
   timeInfo.seconds += seconds;
@@ -42,65 +15,137 @@ function accumulateTime(timeInfo, timeLineEvent) {
     timeInfo.nanoseconds -= 1e9;
   }
 
-  timeInfo.count++;
+  timeInfo.eventCount++;
   return timeInfo;
 }
 
-// We're going to run through each (library, scenario) tuple twice:
-//  1. For each library, run each scenario type
-//  2. For each scenario type, run each library
+function formatTime(timeInfo) {
+  if (timeInfo) {
+    const { seconds, nanoseconds, eventCount } = timeInfo;
+    const duration = `${seconds}.${nanoseconds}`;
+    return duration;
+  }
+  return '---';
+}
+
+// Create a timeline to track each (library,scenario) tuple
+const timelines = allLibraries.map(() => allScenarios.map(() => new Timeline()));
+
+// First, set up each scenario's test cases
+const scenarioCases = allScenarios.map((scenarioInfo) => {
+  const { numIterations } = options;
+  const { id, label, setup } = scenarioInfo;
+
+  if (!setup) {
+    throw new Error(`Cannot set up scenario ${id} (${label}): It has no setup()`);
+  }
+  const cases = setup(numIterations);
+
+  if (!cases || cases.length !== numIterations) {
+    throw new Error(
+      `Cannot set up scenario ${id} (${label}): Its setup() should return ${numIterations} cases. We got ${cases &&
+        cases.length}`,
+    );
+  }
+  return cases;
+});
 
-for (let libNum = 0; libNum < allLibraries.length; libNum++) {
-  allLibraries[libNum].timeline = new Timeline();
+function runTestCases(libraryNum, scenarioNum) {
+  const { numIterations, showLogs, showWarnings } = options;
+  const { label: scenarioLabel, id: scenarioId } = allScenarios[scenarioNum];
+  const { label: libraryLabel, [scenarioId]: libraryFn } = allLibraries[libraryNum];
+  const cases = scenarioCases[scenarioNum];
+  const timeline = timelines[libraryNum][scenarioNum];
+
+  if (libraryFn) {
+    if (showLogs) {
+      console.log(`Running ${scenarioId} (${scenarioLabel}) for ${libraryLabel}`);
+    }
+
+    const event = timeline.startEvent();
+    for (let i = 0; i < numIterations; i++) {
+      libraryFn.apply(null, cases[i]);
+    }
+    event.end();
+  } else if (showWarnings) {
+    console.error(
+      `Library "${libraryLabel}" does not implement the ${scenarioId} (${scenarioLabel}) scenario`,
+    );
+  }
+}
+
+// Here's where we actually run everything
 
+if (options.loopLibrariesThenScenarios) {
+  // For each library, run each scenario
+  for (let libraryNum = 0; libraryNum < allLibraries.length; libraryNum++) {
+    for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) {
+      runTestCases(libraryNum, scenarioNum);
+    }
+  }
+}
+
+if (options.loopScenariosThenLibraries) {
+  // For each scenario, run each library
   for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) {
-    const library = allLibraries[libNum];
-    const scenario = allScenarios[scenarioNum];
-    console.log(`Running ${scenario.displayName} for ${library.label}`);
-    scenario(library);
+    for (let libraryNum = 0; libraryNum < allLibraries.length; libraryNum++) {
+      runTestCases(libraryNum, scenarioNum);
+    }
   }
 }
 
+// Cleanup
 for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) {
-  for (let libNum = 0; libNum < allLibraries.length; libNum++) {
-    allScenarios[scenarioNum](allLibraries[libNum]);
+  const { teardown } = allScenarios[scenarioNum];
+  if (teardown) {
+    const cases = scenarioCases[scenarioNum];
+    teardown(cases);
   }
 }
 
-// Now let's take a look at the results
-
-for (let libNum = 0; libNum < allLibraries.length; libNum++) {
-  const { label: libraryLabel, timeline } = allLibraries[libNum];
-  timeline.end();
-
-  // Average together each scenario's times, as well as the overall time.
-  // Similar to ts-timeframe, seconds and nanoseconds are stored separately for more accurate calculations.
-  const totalTimeByLabel = {};
-  const totalTimeSum = { seconds: 0, nanoseconds: 0, count: 0 };
-  // Start from 1 to skip time that wasn't devoted to any task
-  for (let i = 1; i < timeline.timeLineEvents.length; i++) {
-    const thisEvent = timeline.timeLineEvents[i];
-
-    // Accumulate label time
-    const [scenarioLabel] = thisEvent.getLabels();
-    totalTimeByLabel[scenarioLabel] = totalTimeByLabel[scenarioLabel] || {
-      seconds: 0,
-      nanoseconds: 0,
-      count: 0,
-    };
-    accumulateTime(totalTimeByLabel[scenarioLabel], thisEvent);
-
-    // Accumulate total time
-    accumulateTime(totalTimeSum, thisEvent);
+// Now let's tally up the results
+const libraryResults = [];
+const libraryScenarioResults = [];
+
+for (let libraryNum = 0; libraryNum < allLibraries.length; libraryNum++) {
+  libraryResults[libraryNum] = { seconds: 0, nanoseconds: 0 };
+  libraryScenarioResults[libraryNum] = [];
+
+  for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) {
+    const timeline = timelines[libraryNum][scenarioNum];
+    timeline.end();
+
+    // Count each event in the timeline
+    // Start from 1 to skip time that wasn't devoted to any task
+    for (let i = 1; i < timeline.timeLineEvents.length; i++) {
+      const thisEvent = timeline.timeLineEvents[i];
+
+      libraryResults[libraryNum] = accumulateTime(libraryResults[libraryNum], thisEvent);
+      libraryScenarioResults[libraryNum][scenarioNum] = accumulateTime(
+        libraryScenarioResults[libraryNum][scenarioNum],
+        thisEvent,
+      );
+    }
   }
+}
+
+console.log('==== Benchmark Results ====');
+
+// Finally, output numbers for each library and its scenarios
+for (let libraryNum = 0; libraryNum < allLibraries.length; libraryNum++) {
+  const { label: libraryLabel } = allLibraries[libraryNum];
 
-  // Now output the results
   console.log(`==== ${libraryLabel} ====`);
-  Object.keys(totalTimeByLabel).forEach((scenarioLabel) => {
-    const { seconds, nanoseconds } = totalTimeByLabel[scenarioLabel];
-    console.log(`${scenarioLabel}: ${seconds}.${nanoseconds}`);
-  });
-  const { seconds, nanoseconds } = totalTimeSum;
-  console.log(`Overall: ${seconds}.${nanoseconds}`);
+
+  for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) {
+    const { label: scenarioLabel, id: scenarioId } = allScenarios[scenarioNum];
+
+    const libraryScenarioTime = libraryScenarioResults[libraryNum][scenarioNum];
+    // console.log(`${scenarioId} (${scenarioLabel}): ${formatTime(libraryScenarioTime)}`);
+    console.log(`${formatTime(libraryScenarioTime)} : ${scenarioId} (${scenarioLabel})`);
+  }
+
+  const libraryTime = libraryResults[libraryNum];
+  console.log(`Total: ${formatTime(libraryTime)}`);
 }
 console.log('==== (end of benchmark results) ====');