Introduce OutlierMode

dotnet · May 29, 2018 · 2b5dde0 · 2b5dde0
1 parent 5f08c2e
commit 2b5dde0
Show file tree

Hide file tree

Showing 21 changed files with 268 additions and 80 deletions.
diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroOutliers.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroOutliers.cs
@@ -2,6 +2,7 @@
 using BenchmarkDotNet.Attributes;
 using BenchmarkDotNet.Configs;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Mathematics;
 
 namespace BenchmarkDotNet.Samples.Intro
 {
@@ -13,8 +14,8 @@ private class Config : ManualConfig
             public Config()
             {
                 var jobBase = Job.Default.WithWarmupCount(0).WithTargetCount(10).WithInvocationCount(1).WithUnrollFactor(1);
-                Add(jobBase.WithRemoveOutliers(false).WithId("DontRemoveOutliers"));
-                Add(jobBase.WithRemoveOutliers(true).WithId("RemoveOutliers"));
+                Add(jobBase.WithOutlierMode(OutlierMode.None).WithId("DontRemoveOutliers"));
+                Add(jobBase.WithOutlierMode(OutlierMode.OnlyUpper).WithId("RemoveUpperOutliers"));
             }
         }
 

diff --git a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs
@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Reports;
+using JetBrains.Annotations;
 
 namespace BenchmarkDotNet.Analysers
 {
@@ -12,37 +13,58 @@ public class OutliersAnalyser : AnalyserBase
         public override string Id => "Outliers";
         public static readonly IAnalyser Default = new OutliersAnalyser();
 
-        private OutliersAnalyser()
-        {
-        }
+        private OutliersAnalyser() { }
 
         public override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report, Summary summary)
         {
             var actual = report.AllMeasurements.Where(m => m.IterationMode == IterationMode.MainTarget).ToArray();
             if (actual.IsEmpty())
                 yield break;
             var result = report.AllMeasurements.Where(m => m.IterationMode == IterationMode.Result).ToArray();
-            var actualOutliers = actual.GetStatistics().Outliers;
-            bool removeOutliers = report.Benchmark.Job.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, EngineResolver.Instance); // TODO: improve
+            var outlierMode = report.Benchmark.Job.ResolveValue(AccuracyMode.OutlierModeCharacteristic, EngineResolver.Instance); // TODO: improve            
+            var statistics = actual.GetStatistics();
+            var allOutliers = statistics.AllOutliers;
+            var actualOutliers = statistics.GetActualOutliers(outlierMode);
 
-            if (result.Length + (actualOutliers.Length * (removeOutliers ? 1 : 0)) != actual.Length)
+            if (result.Length + actualOutliers.Length != actual.Length)
             {
                 // This should never happen
                 yield return CreateHint(
-                    string.Format(
-                        "Something went wrong with outliers: Size(MainTarget) = {0}, Size(MainTarget/Outliers) = {1}, Size(Result) = {2}), RemoveOutliers = {3}",
-                        actual.Length, actualOutliers.Length, result.Length, removeOutliers),
+                    $"Something went wrong with outliers: " +
+                    $"Size(MainTarget) = {actual.Length}, " +
+                    $"Size(MainTarget/Outliers) = {actualOutliers.Length}, " +
+                    $"Size(Result) = {result.Length}), " +
+                    $"OutlierMode = {outlierMode}",
                     report);
                 yield break;
             }
 
-            if (actualOutliers.Any())
+            if (allOutliers.Any())
+                yield return CreateHint(GetMessage(actualOutliers.Length, allOutliers.Length), report);
+        }
+
+        /// <summary>
+        /// Returns a nice message which can be displayed in the summary.
+        /// </summary>
+        /// <param name="actualOutliers">Actual outliers which were removed from the statistics</param>
+        /// <param name="allOutliers">All outliers which present in the distribution (lower and upper)</param>
+        /// <returns>The message</returns>
+        [PublicAPI, NotNull, Pure]
+        public static string GetMessage(int actualOutliers, int allOutliers)
+        {
+            string Format(int n, string verb)
             {
-                int n = actualOutliers.Length;
                 string words = n == 1 ? "outlier  was " : "outliers were";
-                string verb = removeOutliers ? "removed" : "detected";
-                yield return CreateHint($"{n} {words} {verb}", report);
+                return $"{n} {words} {verb}";
             }
+
+            if (allOutliers == 0)
+                return string.Empty;
+            if (actualOutliers == allOutliers)
+                return Format(actualOutliers, "removed");
+            if (actualOutliers == 0)
+                return Format(allOutliers, "detected");
+            return Format(actualOutliers, "removed") + ", " + Format(allOutliers, "detected");
         }
     }
 }
diff --git a/src/BenchmarkDotNet/Engines/Engine.cs b/src/BenchmarkDotNet/Engines/Engine.cs
@@ -6,7 +6,6 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Portability;
 using BenchmarkDotNet.Reports;
-using BenchmarkDotNet.Running;
 using JetBrains.Annotations;
 
 namespace BenchmarkDotNet.Engines
@@ -111,9 +110,9 @@ public RunResults Run()
                 ? MeasureGcStats(new IterationData(IterationMode.MainTarget, 0, invokeCount, UnrollFactor)) 
                 : GcStats.Empty;
 
-            bool removeOutliers = TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, Resolver);
+            var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);
 
-            return new RunResults(idle, main, removeOutliers, workGcHasDone);
+            return new RunResults(idle, main, outlierMode, workGcHasDone);
         }
 
         public Measurement RunIteration(IterationData data)

diff --git a/src/BenchmarkDotNet/Engines/EngineResolver.cs b/src/BenchmarkDotNet/Engines/EngineResolver.cs
@@ -2,6 +2,7 @@
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Horology;
 using BenchmarkDotNet.Jobs;
+using BenchmarkDotNet.Mathematics;
 
 namespace BenchmarkDotNet.Engines
 {
@@ -24,16 +25,16 @@ private EngineResolver()
             Register(AccuracyMode.MinIterationTimeCharacteristic, () => TimeInterval.Millisecond * 500);
             Register(AccuracyMode.MinInvokeCountCharacteristic, () => 4);
             Register(AccuracyMode.EvaluateOverheadCharacteristic, () => true);
-            Register(AccuracyMode.RemoveOutliersCharacteristic, job =>
+            Register(AccuracyMode.OutlierModeCharacteristic, job =>
             {
                 var strategy = job.ResolveValue(RunMode.RunStrategyCharacteristic, this);
                 switch (strategy)
                 {
                     case RunStrategy.Throughput:
-                        return true;
+                        return OutlierMode.OnlyUpper;
                     case RunStrategy.ColdStart:
                     case RunStrategy.Monitoring:
-                        return false;
+                        return OutlierMode.None;
                     default:
                         throw new NotSupportedException($"Unknown runStrategy: {strategy}");
                 }

diff --git a/src/BenchmarkDotNet/Engines/EngineTargetStage.cs b/src/BenchmarkDotNet/Engines/EngineTargetStage.cs
@@ -16,7 +16,7 @@ public class EngineTargetStage : EngineStage
         private readonly int? targetCount;
         private readonly double maxRelativeError;
         private readonly TimeInterval? maxAbsoluteError;
-        private readonly bool removeOutliers;
+        private readonly OutlierMode outlierMode;
         private readonly int minIterationCount;
         private readonly int maxIterationCount;
 
@@ -25,7 +25,7 @@ public EngineTargetStage(IEngine engine) : base(engine)
             targetCount = engine.TargetJob.ResolveValueAsNullable(RunMode.TargetCountCharacteristic);
             maxRelativeError = engine.TargetJob.ResolveValue(AccuracyMode.MaxRelativeErrorCharacteristic, engine.Resolver);
             maxAbsoluteError = engine.TargetJob.ResolveValueAsNullable(AccuracyMode.MaxAbsoluteErrorCharacteristic);
-            removeOutliers = engine.TargetJob.ResolveValue(AccuracyMode.RemoveOutliersCharacteristic, engine.Resolver);
+            outlierMode = engine.TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, engine.Resolver);
             minIterationCount = engine.TargetJob.ResolveValue(RunMode.MinTargetIterationCountCharacteristic, engine.Resolver);
             maxIterationCount = engine.TargetJob.ResolveValue(RunMode.MaxTargetIterationCountCharacteristic, engine.Resolver);
         }
@@ -56,7 +56,7 @@ private List<Measurement> RunAuto(long invokeCount, IterationMode iterationMode,
                 measurements.Add(measurement);
                 measurementsForStatistics.Add(measurement);
 
-                var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, removeOutliers);
+                var statistics = MeasurementsStatistics.Calculate(measurementsForStatistics, outlierMode);
                 double actualError = statistics.ConfidenceInterval.Margin;
 
                 double maxError1 = effectiveMaxRelativeError * statistics.Mean;

diff --git a/src/BenchmarkDotNet/Engines/RunResults.cs b/src/BenchmarkDotNet/Engines/RunResults.cs
@@ -10,7 +10,7 @@ namespace BenchmarkDotNet.Engines
 {
     public struct RunResults
     {
-        private readonly bool removeOutliers;
+        private readonly OutlierMode outlierMode;
 
         [CanBeNull]
         public IReadOnlyList<Measurement> Idle { get; }
@@ -21,9 +21,9 @@ public struct RunResults
         public GcStats GCStats { get; }
 
         public RunResults(
-            [CanBeNull] IReadOnlyList<Measurement> idle, [NotNull] IReadOnlyList<Measurement> main, bool removeOutliers, GcStats gcStats)
+            [CanBeNull] IReadOnlyList<Measurement> idle, [NotNull] IReadOnlyList<Measurement> main, OutlierMode outlierMode, GcStats gcStats)
         {
-            this.removeOutliers = removeOutliers;
+            this.outlierMode = outlierMode;
             Idle = idle;
             Main = main;
             GCStats = gcStats;
@@ -36,7 +36,7 @@ public IEnumerable<Measurement> GetMeasurements()
             int resultIndex = 0;
             foreach (var measurement in Main)
             {
-                if (removeOutliers && mainStats.IsOutlier(measurement.Nanoseconds))
+                if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode))
                     continue;
 
                 double value = Math.Max(0, measurement.Nanoseconds - overhead);

diff --git a/src/BenchmarkDotNet/Exporters/Xml/XmlExporterBase.cs b/src/BenchmarkDotNet/Exporters/Xml/XmlExporterBase.cs
@@ -46,7 +46,7 @@ private IXmlSerializer BuildSerializer(Summary summary)
                                                        nameof(Measurement))
                                .WithCollectionItemName(nameof(SummaryDto.Benchmarks),
                                                        nameof(BenchmarkReport.Benchmark))
-                               .WithCollectionItemName(nameof(Statistics.Outliers), "Outlier");
+                               .WithCollectionItemName(nameof(Statistics.AllOutliers), "Outlier");
 
             if (!summary.Config.HasMemoryDiagnoser())
             {

diff --git a/src/BenchmarkDotNet/Jobs/AccuracyMode.cs b/src/BenchmarkDotNet/Jobs/AccuracyMode.cs
@@ -1,5 +1,6 @@
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Mathematics;
 
 namespace BenchmarkDotNet.Jobs
 {
@@ -10,7 +11,7 @@ public sealed class AccuracyMode : JobMode<AccuracyMode>
         public static readonly Characteristic<TimeInterval> MinIterationTimeCharacteristic = CreateCharacteristic<TimeInterval>(nameof(MinIterationTime));
         public static readonly Characteristic<int> MinInvokeCountCharacteristic = CreateCharacteristic<int>(nameof(MinInvokeCount));
         public static readonly Characteristic<bool> EvaluateOverheadCharacteristic = CreateCharacteristic<bool>(nameof(EvaluateOverhead));
-        public static readonly Characteristic<bool> RemoveOutliersCharacteristic = CreateCharacteristic<bool>(nameof(RemoveOutliers));
+        public static readonly Characteristic<OutlierMode> OutlierModeCharacteristic = CreateCharacteristic<OutlierMode>(nameof(OutlierMode));
         public static readonly Characteristic<bool> AnalyzeLaunchVarianceCharacteristic = CreateCharacteristic<bool>(nameof(AnalyzeLaunchVariance));
 
         public double MaxRelativeError
@@ -43,10 +44,10 @@ public bool EvaluateOverhead
             set => EvaluateOverheadCharacteristic[this] = value;
         }
 
-        public bool RemoveOutliers
+        public OutlierMode OutlierMode
         {
-            get => RemoveOutliersCharacteristic[this];
-            set => RemoveOutliersCharacteristic[this] = value;
+            get => OutlierModeCharacteristic[this];
+            set => OutlierModeCharacteristic[this] = value;
         }
 
         public bool AnalyzeLaunchVariance

diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs
@@ -1,10 +1,10 @@
 using System;
 using System.Collections.Generic;
 using BenchmarkDotNet.Toolchains;
-using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Horology;
+using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Running;
 
 namespace BenchmarkDotNet.Jobs
@@ -52,7 +52,9 @@ public static class JobExtensions
         public static Job WithMinIterationTime(this Job job, TimeInterval value) => job.WithCore(j => j.Accuracy.MinIterationTime = value);
         public static Job WithMinInvokeCount(this Job job, int value) => job.WithCore(j => j.Accuracy.MinInvokeCount = value);
         public static Job WithEvaluateOverhead(this Job job, bool value) => job.WithCore(j => j.Accuracy.EvaluateOverhead = value);
-        public static Job WithRemoveOutliers(this Job job, bool value) => job.WithCore(j => j.Accuracy.RemoveOutliers = value);
+        public static Job WithOutlierMode(this Job job, OutlierMode value) => job.WithCore(j => j.Accuracy.OutlierMode = value);
+        [Obsolete("Please use the new WithOutlierMode instead")]
+        public static Job WithRemoveOutliers(this Job job, bool value) => job.WithCore(j => j.Accuracy.OutlierMode = value ? OutlierMode.OnlyUpper : OutlierMode.None);
         public static Job WithAnalyzeLaunchVariance(this Job job, bool value) => job.WithCore(j => j.Accuracy.AnalyzeLaunchVariance = value);
 
         // Meta

diff --git a/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs b/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs
@@ -31,7 +31,7 @@ private MeasurementsStatistics(double standardError, double mean, ConfidenceInte
             ConfidenceInterval = confidenceInterval;
         }
 
-        public static MeasurementsStatistics Calculate(List<Measurement> measurements, bool removeOutliers)
+        public static MeasurementsStatistics Calculate(List<Measurement> measurements, OutlierMode outlierMode)
         {
             int n = measurements.Count;
             if (n == 0)
@@ -45,7 +45,7 @@ public static MeasurementsStatistics Calculate(List<Measurement> measurements, b
             double standardError = standardDeviation / Math.Sqrt(n);
             var confidenceInterval = new ConfidenceInterval(mean, standardError, n);
 
-            if (!removeOutliers) // most simple scenario is done without allocations! but this is not the default case
+            if (outlierMode == OutlierMode.None) // most simple scenario is done without allocations! but this is not the default case
                 return new MeasurementsStatistics(standardError, mean, confidenceInterval);
 
             measurements.Sort(); // sort in place
@@ -65,10 +65,10 @@ public static MeasurementsStatistics Calculate(List<Measurement> measurements, b
             double lowerFence = q1 - 1.5 * interquartileRange;
             double upperFence = q3 + 1.5 * interquartileRange;
 
-            SumWithoutOutliers(measurements, lowerFence, upperFence, out sum, out n); // updates sum and N
+            SumWithoutOutliers(outlierMode, measurements, lowerFence, upperFence, out sum, out n); // updates sum and N
             mean = sum / n;
 
-            variance = VarianceWithoutOutliers(measurements, n, mean, lowerFence, upperFence);
+            variance = VarianceWithoutOutliers(outlierMode, measurements, n, mean, lowerFence, upperFence);
             standardDeviation = Math.Sqrt(variance);
             standardError = standardDeviation / Math.Sqrt(n);
             confidenceInterval = new ConfidenceInterval(mean, standardError, n);
@@ -84,14 +84,14 @@ private static double Sum(List<Measurement> measurements)
             return sum;
         }
 
-        private static void SumWithoutOutliers(List<Measurement> measurements,
+        private static void SumWithoutOutliers(OutlierMode outlierMode, List<Measurement> measurements,
             double lowerFence, double upperFence, out double sum, out int n)
         {
             sum = 0;
             n = 0;
 
             for (int i = 0; i < measurements.Count; i++)
-                if (!IsOutlier(measurements[i].Nanoseconds, lowerFence, upperFence))
+                if (!IsOutlier(outlierMode, measurements[i].Nanoseconds, lowerFence, upperFence))
                 {
                     sum += measurements[i].Nanoseconds;
                     ++n;
@@ -110,14 +110,14 @@ private static double Variance(List<Measurement> measurements, int n, double mea
             return variance;
         }
 
-        private static double VarianceWithoutOutliers(List<Measurement> measurements, int n, double mean, double lowerFence, double upperFence)
+        private static double VarianceWithoutOutliers(OutlierMode outlierMode, List<Measurement> measurements, int n, double mean, double lowerFence, double upperFence)
         {
             if (n == 1)
                 return 0;
 
             double variance = 0;
             for (int i = 0; i < measurements.Count; i++)
-                if (!IsOutlier(measurements[i].Nanoseconds, lowerFence, upperFence))
+                if (!IsOutlier(outlierMode, measurements[i].Nanoseconds, lowerFence, upperFence))
                     variance += (measurements[i].Nanoseconds - mean) * (measurements[i].Nanoseconds - mean) / (n - 1);
 
             return variance;
@@ -131,7 +131,21 @@ private static double GetQuartile(List<Measurement> measurements, int count)
             return measurements[count / 2].Nanoseconds;
         }
 
-        private static bool IsOutlier(double value, double lowerFence, double upperFence)
-            => value < lowerFence || value > upperFence;
+        private static bool IsOutlier(OutlierMode outlierMode, double value, double lowerFence, double upperFence)
+        {
+            switch (outlierMode)
+            {
+                case OutlierMode.None:
+                    return false;
+                case OutlierMode.OnlyUpper:
+                    return value > upperFence;
+                case OutlierMode.OnlyLower:
+                    return value < lowerFence;
+                case OutlierMode.All:
+                    return value < lowerFence || value > upperFence;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(outlierMode), outlierMode, null);
+            }            
+        }
     }
 }
diff --git a/src/BenchmarkDotNet/Mathematics/OutlierMode.cs b/src/BenchmarkDotNet/Mathematics/OutlierMode.cs
@@ -0,0 +1,28 @@
+namespace BenchmarkDotNet.Mathematics
+{
+    /// <summary>
+    /// The enum is design to remove some outliers from the distribution.
+    /// </summary>
+    public enum OutlierMode
+    {
+        /// <summary>
+        /// Don't remove outliers.
+        /// </summary>
+        None,
+
+        /// <summary>
+        /// Remove only upper outliers (which is bigger than upperFence).
+        /// </summary>
+        OnlyUpper,
+
+        /// <summary>
+        /// Remove only lower outliers (which is smaller than lowerFence).
+        /// </summary>
+        OnlyLower,
+
+        /// <summary>
+        /// Remove all outliers.
+        /// </summary>
+        All
+    }
+}