Add outliers configuration property

src/bin/config/run.js

@@ -42,4 +42,20 @@ after the percentage.
     is node.
   - Several limits can be specified at once.`,
   },
+  outliers: {
+    group: TASKS,
+    boolean: true,
+    describe: `Some measures can be extremely slower than others.
+Most of the times, those outliers are due to concurrent logic unrelated to
+your tasks such as garbage collection or OS background processes.
+
+Outliers lower accuracy and significantly slow down benchmarks.
+Therefore, they are ignored by default.
+They can be kept by setting this configuration property to "true" instead.
+
+We recommend keeping the default behavior unless you are confident those
+measures are important for the benchmark's results.
+
+Default: false`,
+  },
 }

src/combination/tasks/find.js

@@ -25,6 +25,7 @@ export const findTasks = async function ({
         precisionTarget: 0,
         cwd,
         previewState: { quiet: true },
+        outliers: true,
         stage: 'init',
         noDimensions,
       },

src/config/default.js

@@ -24,6 +24,7 @@ export const DEFAULT_CONFIG = {
   precision: 5,
   system: 'default_system',
   save: false,
+  outliers: false,
   runnerConfig: {},
   reporterConfig: {},
   limit: [],

src/dev/main.js

@@ -21,6 +21,7 @@ const combinationDev = async function (combination, noDimensions, { cwd }) {
     precisionTarget: 0,
     cwd,
     previewState: { quiet: true },
+    outliers: true,
     stage: 'dev',
     noDimensions,
   })

src/run/main.js

@@ -86,14 +86,15 @@ const previewAndMeasure = async function ({
 
 const measureResult = async function ({
   result,
-  config: { cwd, precisionTarget },
+  config: { cwd, precisionTarget, outliers },
   previewState,
   noDimensions,
 }) {
   const combinations = await measureCombinations(result.combinations, {
     precisionTarget,
     cwd,
     previewState,
+    outliers,
     stage: 'main',
     noDimensions,
   })

src/run/measure/loop.js

@@ -32,6 +32,7 @@ export const performMeasureLoop = async function ({
   previewState,
   stopState,
   stage,
+  outliers,
   server,
   logsFd,
   minLoopDuration,
@@ -44,6 +45,7 @@ export const performMeasureLoop = async function ({
       performSample(state, {
         previewState,
         precisionTarget,
+        outliers,
         server,
         minLoopDuration,
         logsFd,
@@ -66,6 +68,7 @@ const performSample = async function (
   {
     previewState,
     precisionTarget,
+    outliers,
     server,
     minLoopDuration,
     logsFd,
@@ -79,7 +82,12 @@ const performSample = async function (
     { server, minLoopDuration, targetSampleDuration },
     sampleState,
   )
-  const statsA = addStats(stats, sampleStateA, minLoopDuration)
+  const statsA = addStats({
+    stats,
+    sampleState: sampleStateA,
+    minLoopDuration,
+    outliers,
+  })
   const statsB = endRunDuration(startStat, statsA)
   await Promise.all([
     updatePreviewStats({

src/stats/add.js

@@ -43,11 +43,12 @@ export const getInitialStats = function () {
 //        last/current result)
 //   - outliersMin, outliersMax: >=0 <=1
 //   - histogram[*].end: <=1
-export const addStats = function (
+export const addStats = function ({
   stats,
-  { measures, unsortedMeasures, sampleLoops, sampleTimes },
+  sampleState: { measures, unsortedMeasures, sampleLoops, sampleTimes },
   minLoopDuration,
-) {
+  outliers,
+}) {
   if (measures.length === 0) {
     return stats
   }
@@ -57,7 +58,7 @@ export const addStats = function (
     sampleTimes,
     minLoopDuration,
   })
-  const computedStats = computeStats(measures, unsortedMeasures)
+  const computedStats = computeStats(measures, unsortedMeasures, outliers)
   return { ...countStats, ...computedStats }
 }
 

src/stats/compute.js

@@ -1,6 +1,6 @@
 import { getHistogram } from './histogram.js'
 import { getLengthFromLoops } from './length.js'
-import { getOutliersPercentages } from './outliers/main.js'
+import { getOutliersPercentage } from './outliers/main.js'
 import { getPrecisionStats } from './precision.js'
 import { getSortedMedian, getQuantiles } from './quantile.js'
 import { getMean } from './sum.js'
@@ -46,8 +46,8 @@ import { getMean } from './sum.js'
 //    important. However, the mean is a far more useful statistic.
 //  - This would create too many statistics for the average, together with the
 //    mean and the median.
-export const computeStats = function (measures, unsortedMeasures) {
-  const { outliersMin, outliersMax } = getOutliersPercentages(measures)
+export const computeStats = function (measures, unsortedMeasures, outliers) {
+  const { outliersMin, outliersMax } = getOutliersPercentage(measures, outliers)
   const { minIndex, maxIndex, length } = getLengthFromLoops(
     measures.length,
     outliersMin,

src/stats/outliers/main.js

@@ -65,8 +65,25 @@ import { THRESHOLDS } from './threshold.js'
 //  - outliersMin|Max 0 should be possible
 //  - Making only slight changes to the measures should not result in big
 //    changes of outliersMin|Max
-export const getOutliersPercentages = function (measures) {
-  if (measures.length <= 2) {
+// Outliers removal can be skipped by using the `outliers: true` configuration
+// property:
+//  - This is discouraged because:
+//     - Outliers performance is usually unrelated to the task itself,
+//       i.e. they decrease accuracy.
+//     - In many cases, this increases `stdev` a lot, which makes runs much
+//       longer.
+//     - It makes the `envDev` logic not work well.
+//  - However, there are a few cases where outliers are useful:
+//     - When a task is a mix of multiple distributions with very different
+//      speed, but are all significant, such as:
+//        - When a task slows down due to memory allocation but only at
+//          specific thresholds
+//           - `Array.push()` is an example of this
+//        - Tasks with custom units
+//     - When real `min|max` is important
+//     - When debugging
+export const getOutliersPercentage = function (measures, outliers) {
+  if (measures.length <= 2 || outliers) {
     return { outliersMin: 0, outliersMax: 0 }
   }