Merge remote-tracking branch 'origin/master' into fix-uniformint

package.json

@@ -42,7 +42,7 @@
     "@types/node": "^14.0.13",
     "benchmark": "^2.1.4",
     "chalk": "^4.1.0",
-    "console-table-printer": "2.4.11",
+    "console-table-printer": "2.4.35",
     "coveralls": "^3.0.9",
     "fast-check": "^2.0.0",
     "glob": "^7.1.6",

perf/benchmark.cjs

@@ -49,6 +49,17 @@ const argv = yargs(hideBin(process.argv))
     default: 500,
     description: 'Number of samples',
   })
+  .option('allow-local-changes', {
+    type: 'boolean',
+    default: false,
+    description: 'Do not check for local changes, your local changes will be automatically stashed and un-stashed',
+  })
+  .option('print-confidence', {
+    type: 'boolean',
+    default: false,
+    description:
+      'Print 95 % confidence range in reports instead of +X% (increase the number of samples to reduce this range)',
+  })
   .option('verbose', {
     alias: 'v',
     type: 'boolean',
@@ -102,13 +113,35 @@ const prettyName = ({ hash, alias }) => {
 const libName = ({ hash, target }) => {
   return `lib-${hash}-${target}`;
 };
+const formatRatio = (ratio) => {
+  return `${ratio >= 0 ? '+' : ''}${ratio.toFixed(2)} %`;
+};
 
 async function run() {
-  // Check that there is no local changes
-  const { err: gitDiffErr } = await execAsync('git diff-index --quiet HEAD --');
-  if (gitDiffErr && gitDiffErr.code) {
-    console.error(`${chalk.red('ERROR')} Please commit or stash your local changes!`);
-    return;
+  let stashedId = null;
+  if (!argv['allow-local-changes']) {
+    // Check that there is no local changes
+    const { err: gitDiffErr } = await execAsync('git diff-index --quiet HEAD --');
+    if (gitDiffErr && gitDiffErr.code) {
+      console.error(`${chalk.red('ERROR')} Please commit or stash your local changes!`);
+      return;
+    }
+  } else {
+    // Stash local changes
+    const localStashId = `bench-${Math.random().toString(16).substr(2)}`;
+    console.info(`${chalk.cyan('INFO ')} Stashing local changes if any under stash name ${localStashId}`);
+    const { stdout: stashCountBefore } = await execAsync('git stash list | wc -l');
+    const { err: gitStashErr } = await execAsync(`git stash push -m "${localStashId}"`);
+    if (gitStashErr && gitStashErr.code) {
+      console.info(`${chalk.yellow('WARN ')} Something went wrong when trying to stash your changes`);
+    }
+    const { stdout: stashCountAfter } = await execAsync('git stash list | wc -l');
+    if (stashCountBefore != null && stashCountAfter != null && String(stashCountAfter) !== String(stashCountBefore)) {
+      stashedId = localStashId;
+      console.info(`${chalk.cyan('INFO ')} Your local changes have been stashed`);
+    } else {
+      console.info(`${chalk.cyan('INFO ')} No local changes to stash`);
+    }
   }
 
   // Extract current branch
@@ -155,6 +188,15 @@ async function run() {
   } finally {
     // Go back to the original branch
     await execFileAsync('git', ['checkout', currentBranch]);
+
+    if (stashedId !== null) {
+      // Applying stash
+      console.info(`${chalk.cyan('INFO ')} Un-Stashing local changes stored under stash ${stashedId}`);
+      const { err: gitStashErr } = await execAsync(`git stash pop stash^{/${stashedId}}`);
+      if (gitStashErr && gitStashErr.code) {
+        console.info(`${chalk.yellow('WARN ')} Something went wrong when trying to un-stash your changes`);
+      }
+    }
   }
 
   const PRERUN_SAMPLES = Math.floor(argv.samples / 10);
@@ -239,7 +281,7 @@ async function run() {
     configurations.map(([type, lib]) => new Benchmark(test.name(type), () => test.run(lib), benchConf))
   );
   const benchmarkStatsFor = (configurationIndex, testIndex) => {
-    return benchmarks[configurationIndex + testIndex * configurations.length].stats.mean;
+    return benchmarks[configurationIndex + testIndex * configurations.length].stats;
   };
 
   // Simple checks concerning number of calls to the underlying generators
@@ -287,33 +329,49 @@ async function run() {
     const table = new Table({
       columns: [
         { name: 'Name', alignment: 'left' },
-        ...configurations.map(([configName]) => ({ name: configName, alignment: 'right' })),
+        ...configurations.map(([configName]) => ({
+          name: configName,
+          alignment: argv['print-confidence'] ? 'center' : 'right',
+        })),
       ],
     });
     // Find the best and worst configurations
     const [idxWorst, idxBest] = configurations.reduce(
       ([idxWorst, idxBest], _, currentConfigIndex) => {
-        const worst = benchmarkStatsFor(idxWorst, testIndex);
-        const best = benchmarkStatsFor(idxBest, testIndex);
-        const current = benchmarkStatsFor(currentConfigIndex, testIndex);
+        const worst = benchmarkStatsFor(idxWorst, testIndex).mean;
+        const best = benchmarkStatsFor(idxBest, testIndex).mean;
+        const current = benchmarkStatsFor(currentConfigIndex, testIndex).mean;
         return [current > worst ? currentConfigIndex : idxWorst, current < best ? currentConfigIndex : idxBest];
       },
       [0, 0]
     );
     // Add rows
     for (let currentConfigIndex = 0; currentConfigIndex !== configurations.length; ++currentConfigIndex) {
-      const currentBenchMean = benchmarkStatsFor(currentConfigIndex, testIndex);
+      const currentBenchStats = benchmarkStatsFor(currentConfigIndex, testIndex);
+      // [mean - 2 * sigma, mean + 2 * sigma] is 95 %
+      const currentBenchWorst = Math.max(Number.MIN_VALUE, currentBenchStats.mean - 2 * currentBenchStats.deviation);
+      const currentBenchMean = currentBenchStats.mean;
+      const currentBenchBest = currentBenchStats.mean + 2 * currentBenchStats.deviation;
       table.addRow(
         {
           Name: configurations[currentConfigIndex][0],
           ...Object.fromEntries(
             configurations.map((config, configIndex) => {
               if (configIndex === currentConfigIndex) {
-                return [config[0], '-'];
+                return [config[0], '—'];
               }
-              const otherBenchMean = benchmarkStatsFor(configIndex, testIndex);
+              const otherBenchStats = benchmarkStatsFor(configIndex, testIndex);
+              const otherBenchWorst = Math.max(Number.MIN_VALUE, otherBenchStats.mean - 2 * otherBenchStats.deviation);
+              const otherBenchMean = otherBenchStats.mean;
+              const otherBenchBest = otherBenchStats.mean + 2 * otherBenchStats.deviation;
+              const ratioWorst = (100.0 * otherBenchWorst) / currentBenchBest - 100.0;
               const ratio = (100.0 * otherBenchMean) / currentBenchMean - 100.0;
-              return [config[0], `${ratio >= 0 ? '+' : ''}${ratio.toFixed(2)} %`];
+              const ratioBest = (100.0 * otherBenchBest) / currentBenchWorst - 100.0;
+              if (argv['print-confidence']) {
+                return [config[0], `${formatRatio(ratioWorst)} — ${formatRatio(ratioBest)}`]; // ~95% interval
+              } else {
+                return [config[0], formatRatio(ratio)];
+              }
             })
           ),
         },