From 1806812c42d5e02f7d1732ddf89c1aa779a56a6e Mon Sep 17 00:00:00 2001 From: Steven Pautz Date: Sat, 12 Sep 2020 10:26:21 -0400 Subject: [PATCH] chore: Begin adding benchmark scripts (#2) * Groundwork for running multi-library benchmark * Add run-benchmark script --- benchmark/README.md | 28 ++++ benchmark/libraries/immer.js | 9 ++ benchmark/libraries/immutable-assign.js | 9 ++ benchmark/libraries/immutable.js | 9 ++ benchmark/libraries/lodash.js | 12 ++ benchmark/libraries/seamless-immutable.js | 9 ++ benchmark/libraries/tiny-immutable-set.js | 9 ++ benchmark/package.json | 31 ++++ benchmark/run-benchmark.js | 166 ++++++++++++++++++++++ benchmark/yarn.lock | 45 ++++++ 10 files changed, 327 insertions(+) create mode 100644 benchmark/README.md create mode 100644 benchmark/libraries/immer.js create mode 100644 benchmark/libraries/immutable-assign.js create mode 100644 benchmark/libraries/immutable.js create mode 100644 benchmark/libraries/lodash.js create mode 100644 benchmark/libraries/seamless-immutable.js create mode 100644 benchmark/libraries/tiny-immutable-set.js create mode 100644 benchmark/package.json create mode 100644 benchmark/run-benchmark.js create mode 100644 benchmark/yarn.lock diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 0000000..1b14c0c --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,28 @@ +# Benchmark of deep immutable setters + +This project exercises several different utilities for deep-setting immutable data. + +Note that these numbers ONLY reflect `set` operations. Many of these libraries also offer `get` functionality, so +bundle sizes are not directly comparable. These are for reference only. + +## Usage + +To run this locally: + +```shell script +yarn install; +yarn run benchmark; +``` + +## Results + +Last run on (date goes here) + +| Library | Version | Bundle size | Time (lower is better) | +| :------------------------------------------------------------------------------ | ------------: | ----------------------------------------------------------------------------------------------------------------------------------- | ---------------------: | +| [Immer](https://immerjs.github.io/immer/) | `7.0.8` | [![gzip size](https://img.shields.io/bundlephobia/minzip/immer)](https://bundlephobia.com/result?p=immer) | (pending) | +| [Immutable.js](https://immutable-js.github.io/immutable-js/) | `4.0.0-rc.12` | [![gzip size](https://img.shields.io/bundlephobia/minzip/immutable)](https://bundlephobia.com/result?p=immutable) | (pending) | +| [immutable-assign](https://github.com/engineforce/ImmutableAssign) (iassign.js) | `2.1.4` | [![gzip size](https://img.shields.io/bundlephobia/minzip/immutable-assign)](https://bundlephobia.com/result?p=immutable-assign) | (pending) | +| [lodash](https://lodash.com/) (setWith + clone) | `4.17.20` | [![gzip size](https://img.shields.io/bundlephobia/minzip/lodash)](https://bundlephobia.com/result?p=lodash) | (pending) | +| [seamless-immutable](https://github.com/rtfeldman/seamless-immutable) | `7.1.4` | [![gzip size](https://img.shields.io/bundlephobia/minzip/seamless-immutable)](https://bundlephobia.com/result?p=seamless-immutable) | (pending) | +| [tiny-immutable-set](https://github.com/spautz/tiny-immutable-set) | `0.1.0` | [![gzip size](https://img.shields.io/bundlephobia/minzip/tiny-immutable-set)](https://bundlephobia.com/result?p=tiny-immutable-set) | (pending) | diff --git a/benchmark/libraries/immer.js b/benchmark/libraries/immer.js new file mode 100644 index 0000000..b6c65f2 --- /dev/null +++ b/benchmark/libraries/immer.js @@ -0,0 +1,9 @@ +const immer = require('immer'); + +const immerCase = { + label: 'immer', + setWithString: null, + setWithArray: null, +}; + +module.exports = immerCase; diff --git a/benchmark/libraries/immutable-assign.js b/benchmark/libraries/immutable-assign.js new file mode 100644 index 0000000..31bc5a9 --- /dev/null +++ b/benchmark/libraries/immutable-assign.js @@ -0,0 +1,9 @@ +const immutableAssign = require('immutable-assign'); + +const immutableAssignCase = { + label: 'immutable-assign', + setWithString: null, + setWithArray: null, +}; + +module.exports = immutableAssignCase; diff --git a/benchmark/libraries/immutable.js b/benchmark/libraries/immutable.js new file mode 100644 index 0000000..5600481 --- /dev/null +++ b/benchmark/libraries/immutable.js @@ -0,0 +1,9 @@ +const immutable = require('immutable'); + +const immutableCase = { + label: 'immutable.js', + setWithString: null, + setWithArray: null, +}; + +module.exports = immutableCase; diff --git a/benchmark/libraries/lodash.js b/benchmark/libraries/lodash.js new file mode 100644 index 0000000..088092f --- /dev/null +++ b/benchmark/libraries/lodash.js @@ -0,0 +1,12 @@ +const setWith = require('lodash/setWith'); +const clone = require('lodash/clone'); + +const lodashImmutableSet = (obj, path, value) => setWith(clone(obj), path, value, clone); + +const lodashCase = { + label: 'lodash', + setWithString: lodashImmutableSet, + setWithArray: lodashImmutableSet, +}; + +module.exports = lodashCase; diff --git a/benchmark/libraries/seamless-immutable.js b/benchmark/libraries/seamless-immutable.js new file mode 100644 index 0000000..47ebe39 --- /dev/null +++ b/benchmark/libraries/seamless-immutable.js @@ -0,0 +1,9 @@ +const seamlessImmutable = require('seamless-immutable'); + +const seamlessImmutableCase = { + label: 'seamless-immutable', + setWithString: null, + setWithArray: null, +}; + +module.exports = seamlessImmutableCase; diff --git a/benchmark/libraries/tiny-immutable-set.js b/benchmark/libraries/tiny-immutable-set.js new file mode 100644 index 0000000..bcf0d2f --- /dev/null +++ b/benchmark/libraries/tiny-immutable-set.js @@ -0,0 +1,9 @@ +const { set } = require('tiny-immutable-set'); + +const tinyImmutableSetCase = { + label: 'tiny-immutable-set', + setWithString: set, + setWithArray: set, +}; + +module.exports = tinyImmutableSetCase; diff --git a/benchmark/package.json b/benchmark/package.json new file mode 100644 index 0000000..4940255 --- /dev/null +++ b/benchmark/package.json @@ -0,0 +1,31 @@ +{ + "name": "tiny-immutable-set-benchmark", + "private": true, + "version": "0.1.0", + "description": "Benchmarking tool for tiny-immutable-set", + "license": "MIT", + "homepage": "https://github.com/spautz/tiny-immutable-set/benchmark#readme", + "bugs": "https://github.com/spautz/tiny-immutable-set/issues", + "repository": { + "type": "git", + "url": "https://github.com/spautz/tiny-immutable-set.git", + "directory": "benchmark" + }, + "author": { + "name": "Steven Pautz", + "url": "https://stevenpautz.com/" + }, + "sideEffects": false, + "scripts": { + "benchmark": "node ./run-benchmark.js" + }, + "dependencies": { + "@nelsongomes/ts-timeframe": "^0.2.2", + "immer": "^7.0.8", + "immutable": "^4.0.0-rc.12", + "immutable-assign": "^2.1.4", + "lodash": "^4.17.20", + "seamless-immutable": "^7.1.4", + "tiny-immutable-set": "^0.1.0" + } +} diff --git a/benchmark/run-benchmark.js b/benchmark/run-benchmark.js new file mode 100644 index 0000000..b246843 --- /dev/null +++ b/benchmark/run-benchmark.js @@ -0,0 +1,166 @@ +const { Timeline } = require('@nelsongomes/ts-timeframe'); + +const allLibraries = [ + require('./libraries/immer'), + require('./libraries/immutable'), + require('./libraries/immutable-assign'), + require('./libraries/lodash'), + require('./libraries/seamless-immutable'), + require('./libraries/tiny-immutable-set'), +]; + +const allScenarios = [runStringScenario, runArrayScenario, runStringScenario, runArrayScenario]; + +const iterationsPerRun = 50000; + +// An object of arrays of objects, with 5 items at each level +const TEST_OBJECT_BREADTH = 5; +const TEST_OBJECT = {}; +for (let propNum = 0; propNum < TEST_OBJECT_BREADTH; propNum++) { + TEST_OBJECT[`prop${propNum}`] = []; + for (let indexNum = 0; indexNum < TEST_OBJECT_BREADTH; indexNum++) { + const obj = {}; + for (let deepNum = 0; deepNum < TEST_OBJECT_BREADTH; deepNum++) { + obj[`deep${deepNum}`] = deepNum; + } + TEST_OBJECT[`prop${propNum}`].push(obj); + } +} + +////////////////////////////////// +function deepFreeze(object) { + // Retrieve the property names defined on object + var propNames = Object.getOwnPropertyNames(object); + + // Freeze properties before freezing self + + for (let name of propNames) { + let value = object[name]; + + if (value && typeof value === 'object') { + deepFreeze(value); + } + } + + return Object.freeze(object); +} + +deepFreeze(TEST_OBJECT); + +////////////////////////////////// + +function runStringScenario(libraryInfo) { + const { label: libraryLabel, setWithString, timeline } = libraryInfo; + + if (setWithString) { + // Pre-generate all paths + const allPaths = []; + for (let i = 0; i < iterationsPerRun; i++) { + const propNum = i % TEST_OBJECT_BREADTH; + const indexNum = (i / TEST_OBJECT_BREADTH) % TEST_OBJECT_BREADTH; + const deepNum = (i / (TEST_OBJECT_BREADTH * TEST_OBJECT_BREADTH)) % TEST_OBJECT_BREADTH; + allPaths.push(`${propNum}[${indexNum}].${deepNum}`); + } + + const event = timeline.startEvent(['string-path']); + for (let i = 0; i < iterationsPerRun; i++) { + setWithString(TEST_OBJECT, allPaths[i], i); + } + event.end(); + } else { + console.error(`Library "${libraryLabel}" is missing the setWithArray scenario!`); + } +} + +function runArrayScenario(libraryInfo) { + const { libraryLabel, setWithArray, timeline } = libraryInfo; + + if (setWithArray) { + // Pre-generate all paths + const allPaths = []; + for (let i = 0; i < iterationsPerRun; i++) { + const prop = i % TEST_OBJECT_BREADTH; + const index = (i / TEST_OBJECT_BREADTH) % TEST_OBJECT_BREADTH; + const deep = (i / (TEST_OBJECT_BREADTH * TEST_OBJECT_BREADTH)) % TEST_OBJECT_BREADTH; + allPaths.push([prop, index, deep]); + } + + const event = timeline.startEvent(['array-path']); + for (let i = 0; i < iterationsPerRun; i++) { + setWithArray(TEST_OBJECT, allPaths[i], i); + } + event.end(); + } else { + console.error(`Library "${libraryLabel}" is missing the setWithArray scenario!`); + } +} + +function accumulateTime(timeInfo, timeLineEvent) { + const [seconds, nanoseconds] = timeLineEvent.getDurationRaw(); + + timeInfo.seconds += seconds; + timeInfo.nanoseconds += nanoseconds; + while (timeInfo.nanoseconds > 1e9) { + timeInfo.seconds++; + timeInfo.nanoseconds -= 1e9; + } + + timeInfo.count++; + return timeInfo; +} + +// We're going to run through each (library, scenario) tuple twice: +// 1. For each library, run each scenario type +// 2. For each scenario type, run each library + +for (let libNum = 0; libNum < allLibraries.length; libNum++) { + allLibraries[libNum].timeline = new Timeline(); + + for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) { + allScenarios[scenarioNum](allLibraries[libNum]); + } +} + +for (let scenarioNum = 0; scenarioNum < allScenarios.length; scenarioNum++) { + for (let libNum = 0; libNum < allLibraries.length; libNum++) { + allScenarios[scenarioNum](allLibraries[libNum]); + } +} + +// Now let's take a look at the results + +for (let libNum = 0; libNum < allLibraries.length; libNum++) { + const { label: libraryLabel, timeline } = allLibraries[libNum]; + timeline.end(); + + // Average together each scenario's times, as well as the overall time. + // Similar to ts-timeframe, seconds and nanoseconds are stored separately for more accurate calculations. + const totalTimeByLabel = {}; + const totalTimeSum = { seconds: 0, nanoseconds: 0, count: 0 }; + // Start from 1 to skip time that wasn't devoted to any task + for (let i = 1; i < timeline.timeLineEvents.length; i++) { + const thisEvent = timeline.timeLineEvents[i]; + + // Accumulate label time + const [scenarioLabel] = thisEvent.getLabels(); + totalTimeByLabel[scenarioLabel] = totalTimeByLabel[scenarioLabel] || { + seconds: 0, + nanoseconds: 0, + count: 0, + }; + accumulateTime(totalTimeByLabel[scenarioLabel], thisEvent); + + // Accumulate total time + accumulateTime(totalTimeSum, thisEvent); + } + + // Now output the results + console.log(`==== ${libraryLabel} ====`); + Object.keys(totalTimeByLabel).forEach((scenarioLabel) => { + const { seconds, nanoseconds } = totalTimeByLabel[scenarioLabel]; + console.log(`${scenarioLabel}: ${seconds}.${nanoseconds}`); + }); + const { seconds, nanoseconds } = totalTimeSum; + console.log(`Overall: ${seconds}.${nanoseconds}`); +} +console.log('==== (end of benchmark results) ===='); diff --git a/benchmark/yarn.lock b/benchmark/yarn.lock new file mode 100644 index 0000000..a6c5686 --- /dev/null +++ b/benchmark/yarn.lock @@ -0,0 +1,45 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@nelsongomes/ts-timeframe@^0.2.2": + version "0.2.2" + resolved "https://registry.yarnpkg.com/@nelsongomes/ts-timeframe/-/ts-timeframe-0.2.2.tgz#b404c3d1b9a94865c1e8920c9238b5b2da56afed" + integrity sha512-1fFIMuRGi4FNqDBpa2hAdQo6EcVdlwhKlAWnnHSVGByhjF1i20npNhubIXRFfQESL/pVaYOhQ/+gAVzx/TVGkw== + +deep-freeze-strict@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/deep-freeze-strict/-/deep-freeze-strict-1.1.1.tgz#77d0583ca24a69be4bbd9ac2fae415d55523e5b0" + integrity sha1-d9BYPKJKab5LvZrC+uQV1VUj5bA= + +immer@^7.0.8: + version "7.0.8" + resolved "https://registry.yarnpkg.com/immer/-/immer-7.0.8.tgz#41dcbc5669a76500d017bef3ad0d03ce0a1d7c1e" + integrity sha512-XnpIN8PXBBaOD43U8Z17qg6RQiKQYGDGGCIbz1ixmLGwBkSWwmrmx5X7d+hTtXDM8ur7m5OdLE0PiO+y5RB3pw== + +immutable-assign@^2.1.4: + version "2.1.4" + resolved "https://registry.yarnpkg.com/immutable-assign/-/immutable-assign-2.1.4.tgz#e040d669a392e20b89cd148e4ef30f49d4e2d752" + integrity sha512-MQPihUVP0/LINARUF3lRoviRQaQgVsX3vPk351epg+qxUplY9MqPMvTTDY6yk7cxLB+oMHmboJND9MaPp05h+Q== + optionalDependencies: + deep-freeze-strict "^1.1.1" + +immutable@^4.0.0-rc.12: + version "4.0.0-rc.12" + resolved "https://registry.yarnpkg.com/immutable/-/immutable-4.0.0-rc.12.tgz#ca59a7e4c19ae8d9bf74a97bdf0f6e2f2a5d0217" + integrity sha512-0M2XxkZLx/mi3t8NVwIm1g8nHoEmM9p9UBl/G9k4+hm0kBgOVdMV/B3CY5dQ8qG8qc80NN4gDV4HQv6FTJ5q7A== + +lodash@^4.17.20: + version "4.17.20" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" + integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== + +seamless-immutable@^7.1.4: + version "7.1.4" + resolved "https://registry.yarnpkg.com/seamless-immutable/-/seamless-immutable-7.1.4.tgz#6e9536def083ddc4dea0207d722e0e80d0f372f8" + integrity sha512-XiUO1QP4ki4E2PHegiGAlu6r82o5A+6tRh7IkGGTVg/h+UoeX4nFBeCGPOhb4CYjvkqsfm/TUtvOMYC1xmV30A== + +tiny-immutable-set@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/tiny-immutable-set/-/tiny-immutable-set-0.1.0.tgz#0d44409eaf5270d52091c328038049bb82511267" + integrity sha512-BySDKZ6JNsXr/u6GnopnkSsYMCqY0TFlxK7laJpvz1LXr3RDrR71Abkh8621n70l8FyuJ+DGzo5fUK35tbwNOg==