Merge c250d2f into a39ab4c

CHANGELOG.md

@@ -1,5 +1,10 @@
 CHANGELOG
 
+v3.x
+- `Bench.addAsync` Experimental `options._throughput` to measure parallel async processing throughput
+- `Bench.add/addAsync` New `options.multiplier` to adjust bench result numbers to account for multiple repetitions per iteration
+- `Bench.add/addAsync` New `options.units` to customize bench result messages
+
 v3.2.0-beta.3 - Dec 5, 2019
 
 - fix transpilation (#117)

docs/api-reference/bench/bench.md

@@ -39,18 +39,18 @@ Adds a test case. Supports multiple signatures:
 
 Parameters
 
-* `priority`=`0` (Number, optional) - allows controlling which bench cases execute. Can also be specified through the `options` object.
 * `id` (String) - The unique string for this test. Used as the description of the test in the results.
-* `initFunc` (Function, options) - Function run before any tests
-* `testFunc` (Function, options) - Function run for each test iteration. 
+* `testFunc` (Function, options) - Function run for each test iteration.
 
 Options
 
-* `initialize`: `() => any` initialization function called once before `testFunc` iterations start.
-* `repetitions` multiplier applied to the number of actual repetitions. Use this if each test case already performs a number of iterations
-* `priority` can be specified as part of options
+* `priority`=`0` (Number, optional) - allows controlling which bench cases execute. Can also be specified through the `options` object.
+* `initialize`=: `() => any` initialization function called once before `testFunc` iterations start.
+* `multiplier`=`1` : `Number` Multiplier applied to the number of actual iterations. Use this if each test case already performs a number of iterations. Affects reporting only.
+* `unit`=`'iterations'`: Can be used to customize the output message for bench results.
+* `_throughput`=`false` : `Number` Use with `Bench.addAsync` to specify that `_throughput` iterations should be run in parallel. Note that automatic iteration selection is not available in this case.
 
-Returns itself for chaining.
+Returns: itself for chaining.
 
 ## addAsync
 

docs/whats-new.md

@@ -1,11 +1,11 @@
 # What's New
 
-## v3.x 
+## v3.x
 
 Release Date: TBD
 
 ### **@probe.gl/bench**
-- New bench case option: `repetitions`
+- New bench case options: `options.multiplier`, `options.units`, `options._throughput`
 
 ## v3.2
 

modules/bench/src/bench-loggers.js

@@ -33,14 +33,14 @@ export function logResultsAsMarkdownTable({entry, id, itersPerSecond, error, tim
   }
 }
 
-export function logResultsAsTree({entry, id, itersPerSecond, error, time}) {
+export function logResultsAsTree({entry, id, itersPerSecond, error, time, unit}) {
   switch (entry) {
     case LOG_ENTRY.GROUP:
       console.log('');
       console.log(`${id}`);
       break;
     case LOG_ENTRY.TEST:
-      console.log(`├─ ${id}: ${itersPerSecond} iterations/s ±${(error * 100).toFixed(2)}%`);
+      console.log(`├─ ${id}: ${itersPerSecond} ${unit}/s ±${(error * 100).toFixed(2)}%`);
       break;
     case LOG_ENTRY.COMPLETE:
       console.log('');
@@ -50,11 +50,11 @@ export function logResultsAsTree({entry, id, itersPerSecond, error, time}) {
   }
 }
 
-export function logResultsAsTreeWithElapsed({entry, id, itersPerSecond, error, time}) {
+export function logResultsAsTreeWithElapsed({entry, id, itersPerSecond, error, time, unit}) {
   switch (entry) {
     case LOG_ENTRY.TEST:
       console.log(
-        `├─ ${id}: ${itersPerSecond} iterations/s ±${(error * 100).toFixed(2)}% (${time.toFixed(
+        `├─ ${id}: ${itersPerSecond} ${unit}/s ±${(error * 100).toFixed(2)}% (${time.toFixed(
           2
         )}s elapsed)`
       );

modules/bench/src/bench.js

@@ -145,7 +145,8 @@ export default class Bench {
     // Test case options
     const opts = {
       ...this.opts,
-      repetitions: 1, // repetitions per test case
+      multiplier: 1, // multiplier per test case
+      unit: 'iterations',
       ...options
     };
 
@@ -196,7 +197,7 @@ async function runTests({tests, onBenchmarkComplete = noop}) {
   for (const id in tests) {
     const test = tests[id];
     if (test.group) {
-      logEntry(test, {entry: LOG_ENTRY.GROUP, id: test.id, message: test.id});
+      logEntry(test, {...test.opts, entry: LOG_ENTRY.GROUP, message: test.id});
     } else {
       await runTest({test, onBenchmarkComplete});
     }
@@ -243,7 +244,13 @@ async function runBenchTestAsync(test) {
   let totalIterations = 0;
 
   for (let i = 0; i < test.opts.minIterations; i++) {
-    const {time, iterations} = await runBenchTestTimedAsync(test, test.opts.time);
+    let time;
+    let iterations;
+    if (test.async && test._throughput) {
+      ({time, iterations} = await runBenchTestParallelIterationsAsync(test, test.throughput));
+    } else {
+      ({time, iterations} = await runBenchTestForMinimumTimeAsync(test, test.opts.time));
+    }
     const iterationsPerSecond = iterations / time;
     results.push(iterationsPerSecond);
     totalTime += time;
@@ -259,7 +266,7 @@ async function runBenchTestAsync(test) {
 }
 
 // Run a test func for an increasing amount of iterations until time threshold exceeded
-async function runBenchTestTimedAsync(test, minTime) {
+async function runBenchTestForMinimumTimeAsync(test, minTime) {
   let iterations = 1;
   let elapsedMillis = 0;
 
@@ -283,24 +290,31 @@ async function runBenchTestTimedAsync(test, minTime) {
 
   return {
     time,
-    // Test cases can use `options.repetitions` to account for multiple repetitions per iteration
-    iterations: iterations * test.opts.repetitions
+    // Test cases can use `options.multiplier` to account for multiple repetitions per iteration
+    iterations: iterations * test.opts.multiplier
   };
 }
 
+// Run a test func for a specific amount of parallel iterations
+async function runBenchTestParallelIterationsAsync(test, iterations) {
+  const testArgs = test.initFunc && test.initFunc();
+
+  const promises = [];
+
+  const {context, testFunc} = test;
+  for (let i = 0; i < iterations; i++) {
+    promises.push(testFunc.call(context, testArgs));
+  }
+
+  return await Promise.all(promises);
+}
+
 // Run a test func for a specific amount of iterations
 async function runBenchTestIterationsAsync(test, iterations) {
   const testArgs = test.initFunc && test.initFunc();
-
   const {context, testFunc} = test;
-  if (context && testArgs) {
-    for (let i = 0; i < iterations; i++) {
-      await testFunc.call(context, testArgs);
-    }
-  } else {
-    for (let i = 0; i < iterations; i++) {
-      await testFunc.call(context);
-    }
+  for (let i = 0; i < iterations; i++) {
+    await testFunc.call(context, testArgs);
   }
 }
 
@@ -310,6 +324,7 @@ async function runBenchTestIterationsAsync(test, iterations) {
 function runBenchTestIterations(test, iterations) {
   const testArgs = test.initFunc && test.initFunc();
 
+  // When running sync, avoid overhead of parameter passing if not needed
   const {context, testFunc} = test;
   if (context && testArgs) {
     for (let i = 0; i < iterations; i++) {

modules/bench/test/iterator.bench.js

@@ -17,32 +17,35 @@ async function* asyncIterator(length) {
 export default function coreBench(bench) {
   return bench
     .group('Core Module - Async Iteration')
-    .add('for (let i=0; ...)', {repetitions: LENGTH}, () => {
+    .add('for (let i=0; ...)', {multiplier: LENGTH}, () => {
       let sum = 0;
       for (let i = 0; i < LENGTH; i++) {
         sum += i;
       }
       return sum;
     })
-    .add('for (const element of ARRAY)', {repetitions: LENGTH}, () => {
+    .add('for (const element of ARRAY)', {multiplier: LENGTH}, () => {
       let sum = 0;
       for (const element of ARRAY) {
         sum += element;
       }
       return sum;
     })
-    .add('for (const element of iterator)', {repetitions: LENGTH}, () => {
+    .add('for (const element of iterator)', {multiplier: LENGTH}, () => {
       let sum = 0;
       for (const element of iterator(LENGTH)) {
         sum += element;
       }
       return sum;
     })
-    .addAsync('for await (const element of asyncIterator)', {repetitions: LENGTH}, async () => {
+    .addAsync('for await (const element of asyncIterator)', {multiplier: LENGTH}, async () => {
       let sum = 0;
       for await (const element of asyncIterator(ARRAY)) {
         sum += element;
       }
       return sum;
+    })
+    .addAsync('parallel wait for 1000 promises', {_throughput: LENGTH}, async () => {
+      return await 1;
     });
 }