Rename OutlierMode values (#1199)

docs/articles/configs/jobs.md

@@ -64,7 +64,7 @@ If you want to change the accuracy level, you should use the following character
 * `MinIterationTime`: Minimum time of a single iteration. Unlike `Run.IterationTime`, this characteristic specifies only the lower limit. In case of need, BenchmarkDotNet can increase this value.
 * `MinInvokeCount`:  Minimum about of target method invocation. Default value if `4` but you can decrease this value for cases when single invocations takes a lot of time.
 * `EvaluateOverhead`: if you benchmark method takes nanoseconds, BenchmarkDotNet overhead can significantly affect measurements. If this characterics is enable, the overhead will be evaluated and subtracted from the result measurements. Default value is `true`.
-* `WithOutlierMode`: sometimes you could have outliers in your measurements. Usually these are unexpected outliers which arose because of other processes activities. By default (`OutlierMode.OnlyUpper`), all upper outliers (which is larger than Q3) will be removed from the result measurements. However, some of benchmarks have *expected* outliers. In these situation, you expect that some of invocation can produce outliers measurements (e.g. in case of network activities, cache operations, and so on). If you want to see result statistics with these outliers, you should use `OutlierMode.None`. If you can also choose `OutlierMode.OnlyLower` (outliers which are smaller than Q1 will be removed) or `OutlierMode.All` (all outliers will be removed). See also: @BenchmarkDotNet.Mathematics.OutlierMode
+* `WithOutlierMode`: sometimes you could have outliers in your measurements. Usually these are unexpected outliers which arose because of other processes activities. By default (`OutlierMode.RemoveUpper`), all upper outliers (which is larger than Q3) will be removed from the result measurements. However, some of benchmarks have *expected* outliers. In these situation, you expect that some of invocation can produce outliers measurements (e.g. in case of network activities, cache operations, and so on). If you want to see result statistics with these outliers, you should use `OutlierMode.DontRemove`. If you can also choose `OutlierMode.RemoveLower` (outliers which are smaller than Q1 will be removed) or `OutlierMode.RemoveAll` (all outliers will be removed). See also: @BenchmarkDotNet.Mathematics.OutlierMode
 * `AnalyzeLaunchVariance`: this characteristic makes sense only if `Run.LaunchCount` is default. If this mode is enabled and, BenchmarkDotNet will try to perform several launches and detect if there is a veriance between launches. If this mode is disable, only one launch will be performed.
 
 ### Infrastructure

docs/articles/guides/console-args.md

@@ -197,7 +197,7 @@ dotnet run -c Release -- --filter * --runtimes netcoreapp2.0 netcoreapp2.1 --sta
 * `-e`, `--exporters` GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML
 * `-i`, `--inProcess` (Default: false) Run benchmarks in Process
 * `-a`, `--artifacts` Valid path to accessible directory
-* `--outliers` (Default: OnlyUpper) None/OnlyUpper/OnlyLower/All
+* `--outliers` (Default: RemoveUpper) DontRemove/RemoveUpper/RemoveLower/RemoveAll
 * `--affinity` Affinity mask to set for the benchmark process
 * `--allStats` (Default: false) Displays all statistics (min, max & more)
 * `--attribute` Run all methods with given attribute (applied to class or method)

docs/articles/samples/IntroOutliers.md

@@ -13,8 +13,8 @@ uid: BenchmarkDotNet.Samples.IntroOutliers
 ```markdown
  Method |                 Job | OutlierMode |     Mean |       Error |      StdDev |
 ------- |-------------------- |------------ |---------:|------------:|------------:|
-    Foo |  DontRemoveOutliers |        None | 150.5 ms | 239.1911 ms | 158.2101 ms |
-    Foo | RemoveUpperOutliers |   OnlyUpper | 100.5 ms |   0.1931 ms |   0.1149 ms |
+    Foo |  DontRemoveOutliers |  DontRemove | 150.5 ms | 239.1911 ms | 158.2101 ms |
+    Foo | RemoveUpperOutliers | RemoveUpper | 100.5 ms |   0.1931 ms |   0.1149 ms |
 
 // * Hints *
 Outliers

samples/BenchmarkDotNet.Samples/IntroOutliers.cs

@@ -14,8 +14,8 @@ private class Config : ManualConfig
             public Config()
             {
                 var jobBase = Job.Default.WithWarmupCount(0).WithIterationCount(10).WithInvocationCount(1).WithUnrollFactor(1);
-                Add(jobBase.WithOutlierMode(OutlierMode.None).WithId("DontRemoveOutliers"));
-                Add(jobBase.WithOutlierMode(OutlierMode.OnlyUpper).WithId("RemoveUpperOutliers"));
+                Add(jobBase.WithOutlierMode(OutlierMode.DontRemove).WithId("DontRemoveOutliers"));
+                Add(jobBase.WithOutlierMode(OutlierMode.RemoveUpper).WithId("RemoveUpperOutliers"));
             }
         }
 

samples/BenchmarkDotNet.Samples/IntroRatioSD.cs

@@ -6,7 +6,7 @@
 namespace BenchmarkDotNet.Samples
 {
     // Don't remove outliers
-    [Outliers(OutlierMode.None)]
+    [Outliers(OutlierMode.DontRemove)]
     // Skip jitting, pilot, warmup; measure 10 iterations
     [SimpleJob(RunStrategy.Monitoring, targetCount: 10, invocationCount: 1)]
     public class IntroRatioSD

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -44,7 +44,7 @@ public class CommandLineOptions
         [Option('a', "artifacts", Required = false, HelpText = "Valid path to accessible directory")]
         public DirectoryInfo ArtifactsDirectory { get; set; }
 
-        [Option("outliers", Required = false, Default = OutlierMode.OnlyUpper, HelpText = "None/OnlyUpper/OnlyLower/All")]
+        [Option("outliers", Required = false, Default = OutlierMode.RemoveUpper, HelpText = "DontRemove/RemoveUpper/RemoveLower/RemoveAll")]
         public OutlierMode Outliers { get; set; }
 
         [Option("affinity", Required = false, HelpText = "Affinity mask to set for the benchmark process")]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -237,7 +237,7 @@ private static Job GetBaseJob(CommandLineOptions options, IConfig globalConfig)
                 globalConfig?.GetJobs().SingleOrDefault(job => job.Meta.IsDefault) // global config might define single custom Default job
                 ?? AvailableJobs[options.BaseJob.ToLowerInvariant()];
 
-            if (baseJob != Job.Dry && options.Outliers != OutlierMode.OnlyUpper)
+            if (baseJob != Job.Dry && options.Outliers != OutlierMode.RemoveUpper)
                 baseJob = baseJob.WithOutlierMode(options.Outliers);
 
             if (options.Affinity.HasValue)

src/BenchmarkDotNet/Engines/EngineResolver.cs

@@ -38,10 +38,10 @@ private EngineResolver()
                 switch (strategy)
                 {
                     case RunStrategy.Throughput:
-                        return OutlierMode.OnlyUpper;
+                        return OutlierMode.RemoveUpper;
                     case RunStrategy.ColdStart:
                     case RunStrategy.Monitoring:
-                        return OutlierMode.None;
+                        return OutlierMode.DontRemove;
                     default:
                         throw new NotSupportedException($"Unknown runStrategy: {strategy}");
                 }

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -320,7 +320,7 @@ public static Job WithEnvironmentVariable(this Job job, [NotNull] string key, [N
         [Obsolete("Please use the new WithOutlierMode instead")]
         [PublicAPI]
         public static Job WithRemoveOutliers(this Job job, bool value) =>
-            job.WithCore(j => j.Accuracy.OutlierMode = value ? OutlierMode.OnlyUpper : OutlierMode.None);
+            job.WithCore(j => j.Accuracy.OutlierMode = value ? OutlierMode.RemoveUpper : OutlierMode.DontRemove);
 
         [PublicAPI]
         public static Job WithAnalyzeLaunchVariance(this Job job, bool value) => job.WithCore(j => j.Accuracy.AnalyzeLaunchVariance = value);

src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs

@@ -47,7 +47,7 @@ public static MeasurementsStatistics Calculate(List<Measurement> measurements, O
             double standardError = standardDeviation / Math.Sqrt(n);
             var confidenceInterval = new ConfidenceInterval(mean, standardError, n);
 
-            if (outlierMode == OutlierMode.None) // most simple scenario is done without allocations! but this is not the default case
+            if (outlierMode == OutlierMode.DontRemove) // most simple scenario is done without allocations! but this is not the default case
                 return new MeasurementsStatistics(standardError, mean, confidenceInterval);
 
             measurements.Sort(); // sort in place
@@ -136,14 +136,20 @@ private static bool IsOutlier(OutlierMode outlierMode, double value, double lowe
         {
             switch (outlierMode)
             {
+#pragma warning disable 618
                 case OutlierMode.None:
+                case OutlierMode.DontRemove:
                     return false;
                 case OutlierMode.OnlyUpper:
+                case OutlierMode.RemoveUpper:
                     return value > upperFence;
                 case OutlierMode.OnlyLower:
+                case OutlierMode.RemoveLower:
                     return value < lowerFence;
                 case OutlierMode.All:
+                case OutlierMode.RemoveAll:
                     return value < lowerFence || value > upperFence;
+#pragma warning restore 618
                 default:
                     throw new ArgumentOutOfRangeException(nameof(outlierMode), outlierMode, null);
             }            

src/BenchmarkDotNet/Mathematics/OutlierMode.cs

@@ -1,4 +1,6 @@
-namespace BenchmarkDotNet.Mathematics
+using System;
+
+namespace BenchmarkDotNet.Mathematics
 {
     /// <summary>
     /// The enum is design to remove some outliers from the distribution.
@@ -8,21 +10,45 @@ public enum OutlierMode
         /// <summary>
         /// Don't remove outliers.
         /// </summary>
+        [Obsolete("Use DontRemoveOutliers")]
         None,
 
         /// <summary>
         /// Remove only upper outliers (which is bigger than upperFence).
         /// </summary>
+        [Obsolete("Use RemoveUpperOutliers")]
         OnlyUpper,
 
         /// <summary>
         /// Remove only lower outliers (which is smaller than lowerFence).
         /// </summary>
+        [Obsolete("Use RemoveUpperOutliers")]
         OnlyLower,
 
         /// <summary>
         /// Remove all outliers.
         /// </summary>
-        All
+        [Obsolete("Use RemoveAllOutliers")]
+        All,
+
+        /// <summary>
+        /// Don't remove outliers.
+        /// </summary>
+        DontRemove,
+
+        /// <summary>
+        /// Remove only upper outliers (which is bigger than upperFence).
+        /// </summary>
+        RemoveUpper,
+
+        /// <summary>
+        /// Remove only lower outliers (which is smaller than lowerFence).
+        /// </summary>
+        RemoveLower,
+
+        /// <summary>
+        /// Remove all outliers.
+        /// </summary>
+        RemoveAll
     }
 }

src/BenchmarkDotNet/Mathematics/Statistics.cs

@@ -91,14 +91,20 @@ public bool IsActualOutlier(double value, OutlierMode outlierMode)
         {
             switch (outlierMode)
             {
+#pragma warning disable 618
                 case OutlierMode.None:
+                case OutlierMode.DontRemove:
                     return false;
                 case OutlierMode.OnlyUpper:
+                case OutlierMode.RemoveUpper:
                     return IsUpperOutlier(value);
                 case OutlierMode.OnlyLower:
+                case OutlierMode.RemoveLower:
                     return IsLowerOutlier(value);
                 case OutlierMode.All:
+                case OutlierMode.RemoveAll:
                     return IsOutlier(value);
+#pragma warning restore 618
                 default:
                     throw new ArgumentOutOfRangeException(nameof(outlierMode), outlierMode, null);
             }
@@ -109,14 +115,20 @@ public double[] GetActualOutliers(OutlierMode outlierMode)
         {
             switch (outlierMode)
             {
+#pragma warning disable 618
                 case OutlierMode.None:
+                case OutlierMode.DontRemove:
                     return Array.Empty<double>();
                 case OutlierMode.OnlyUpper:
+                case OutlierMode.RemoveUpper:
                     return UpperOutliers;
                 case OutlierMode.OnlyLower:
+                case OutlierMode.RemoveLower:
                     return LowerOutliers;
                 case OutlierMode.All:
+                case OutlierMode.RemoveAll:
                     return AllOutliers;
+#pragma warning restore 618
                 default:
                     throw new ArgumentOutOfRangeException(nameof(outlierMode), outlierMode, null);
             }

tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs

@@ -78,7 +78,7 @@ public RunResults Run()
                 return new RunResults(
                     new List<Measurement> { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) },
                     new List<Measurement> { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) },
-                    OutlierMode.None,
+                    OutlierMode.DontRemove,
                     default,
                     default);
             }

tests/BenchmarkDotNet.Tests/Engine/EngineResultStageTests.cs

@@ -21,11 +21,11 @@ public void OutliersTest()
             Add(measurements, 104);
             Add(measurements, 500); // It's an outlier
 
-            CheckResults(5, measurements, OutlierMode.OnlyUpper);
-            CheckResults(5, measurements, OutlierMode.All);
+            CheckResults(5, measurements, OutlierMode.RemoveUpper);
+            CheckResults(5, measurements, OutlierMode.RemoveAll);
 
-            CheckResults(6, measurements, OutlierMode.None);
-            CheckResults(6, measurements, OutlierMode.OnlyLower);
+            CheckResults(6, measurements, OutlierMode.DontRemove);
+            CheckResults(6, measurements, OutlierMode.RemoveLower);
         }
 
         [AssertionMethod]

tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs

@@ -188,12 +188,12 @@ public void FewMutatorsCanBeAppliedToSameType()
 
             Assert.Equal(1, benchmarkCase.Job.Run.InvocationCount);
             Assert.Equal(1, benchmarkCase.Job.Run.UnrollFactor);
-            Assert.Equal(OutlierMode.None, benchmarkCase.Job.Accuracy.OutlierMode);
+            Assert.Equal(OutlierMode.DontRemove, benchmarkCase.Job.Accuracy.OutlierMode);
             Assert.False(benchmarkCase.Job.Meta.IsMutator);
         }
 
         [RunOncePerIteration]
-        [Outliers(OutlierMode.None)]
+        [Outliers(OutlierMode.DontRemove)]
         public class WithFewMutators
         {
             [Benchmark] public void Method() { }