Bump Perfolizer: 0.2.1->0.3.16

dotnet · Mar 9, 2024 · 3062103 · 3062103
1 parent 3305b34
commit 3062103
Show file tree

Hide file tree

Showing 84 changed files with 1,086 additions and 1,068 deletions.
diff --git a/BenchmarkDotNet.sln.DotSettings b/BenchmarkDotNet.sln.DotSettings
@@ -52,6 +52,8 @@
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=OSX/@EntryIndexedValue">OSX</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=RT/@EntryIndexedValue">RT</s:String>
 	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=4a98fdf6_002D7d98_002D4f5a_002Dafeb_002Dea44ad98c70c/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Instance" AccessRightKinds="Private" Description="Instance fields (private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;Kind Name="READONLY_FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
+	<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/UserRules/=f9fce829_002De6f4_002D4cb2_002D80f1_002D5497c44f51df/@EntryIndexedValue">&lt;Policy&gt;&lt;Descriptor Staticness="Static" AccessRightKinds="Private" Description="Static fields (private)"&gt;&lt;ElementKinds&gt;&lt;Kind Name="FIELD" /&gt;&lt;/ElementKinds&gt;&lt;/Descriptor&gt;&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;&lt;/Policy&gt;</s:String>
 	<s:Boolean x:Key="/Default/Environment/Filtering/ExcludeCoverageFilters/=_002A_003B_002A_003BJetBrains_002EAnnotations_002E_002A_003B_002A/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/Environment/Filtering/ExcludeCoverageFilters/=_002A_003B_002A_003BSimpleJson_002E_002A_003B_002A/@EntryIndexedValue">True</s:Boolean>
 	<s:Int64 x:Key="/Default/Environment/Hierarchy/Build/BuildTool/MsbuildVersion/@EntryValue">983040</s:Int64>
@@ -68,6 +70,7 @@
 	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EMigrateThisQualifierSettings/@EntryIndexedValue">True</s:Boolean>
     <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateInstanceFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
     <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/PredefinedNamingRules/=PrivateStaticFields/@EntryIndexedValue">&lt;Policy Inspect="True" Prefix="" Suffix="" Style="aaBb" /&gt;</s:String>
+	<s:Boolean x:Key="/Default/Environment/SettingsMigration/IsMigratorApplied/=JetBrains_002EReSharper_002EPsi_002ECSharp_002ECodeStyle_002ESettingsUpgrade_002EPredefinedNamingRulesToUserRulesUpgrade/@EntryIndexedValue">True</s:Boolean>
 	<s:String x:Key="/Default/FilterSettingsManager/CoverageFilterXml/@EntryValue">&lt;data&gt;&lt;IncludeFilters /&gt;&lt;ExcludeFilters&gt;&lt;Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="JetBrains.Annotations.*" FunctionMask="*" IsEnabled="True" /&gt;&lt;Filter ModuleMask="*" ModuleVersionMask="*" ClassMask="SimpleJson.*" FunctionMask="*" IsEnabled="True" /&gt;&lt;/ExcludeFilters&gt;&lt;/data&gt;</s:String>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Acpi/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Affinitize/@EntryIndexedValue">True</s:Boolean>
@@ -162,6 +165,7 @@
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=parameterless/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Partitioner/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=pdbonly/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=Perfolizer/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=poco/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Prerelease/@EntryIndexedValue">True</s:Boolean>
 	<s:Boolean x:Key="/Default/UserDictionary/Words/=Prettifier/@EntryIndexedValue">True</s:Boolean>

diff --git a/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs b/samples/BenchmarkDotNet.Samples/IntroStatisticalTesting.cs
@@ -1,14 +1,10 @@
 using System.Threading;
 using BenchmarkDotNet.Attributes;
-using Perfolizer.Mathematics.SignificanceTesting;
-using Perfolizer.Mathematics.Thresholds;
 
 namespace BenchmarkDotNet.Samples
 {
-    [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Microseconds, 1, true)]
-    [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Microseconds, 1, true)]
-    [StatisticalTestColumn(StatisticalTestKind.Welch, ThresholdUnit.Ratio, 0.03, true)]
-    [StatisticalTestColumn(StatisticalTestKind.MannWhitney, ThresholdUnit.Ratio, 0.03, true)]
+    [StatisticalTestColumn("1us")]
+    [StatisticalTestColumn("3%")]
     [SimpleJob(warmupCount: 0, iterationCount: 5)]
     public class IntroStatisticalTesting
     {

diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/JitStatsDiagnoser.cs
@@ -6,6 +6,7 @@
 using BenchmarkDotNet.Running;
 using Microsoft.Diagnostics.Tracing.Parsers;
 using Microsoft.Diagnostics.Tracing.Session;
+using Perfolizer.Metrology;
 
 namespace BenchmarkDotNet.Diagnostics.Windows
 {
@@ -97,7 +98,7 @@ private sealed class JitAllocatedMemoryDescriptor : IMetricDescriptor
             public bool TheGreaterTheBetter => false;
             public string NumberFormat => "N0";
             public UnitType UnitType => UnitType.Size;
-            public string Unit => SizeUnit.B.Name;
+            public string Unit => SizeUnit.B.Abbreviation;
             public int PriorityInCategory => 0;
             public bool GetIsAvailable(Metric metric) => true;
         }

diff --git a/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs b/src/BenchmarkDotNet.Diagnostics.Windows/Tracing/NativeMemoryLogParser.cs
@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
-using BenchmarkDotNet.Columns;
 using BenchmarkDotNet.Diagnosers;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Extensions;
@@ -12,6 +11,7 @@
 using Microsoft.Diagnostics.Tracing.Etlx;
 using Microsoft.Diagnostics.Tracing.Parsers.Kernel;
 using Microsoft.Diagnostics.Tracing.Stacks;
+using Perfolizer.Metrology;
 using Address = System.UInt64;
 
 namespace BenchmarkDotNet.Diagnostics.Windows.Tracing
@@ -246,12 +246,14 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
             var memoryAllocatedPerOperation = totalAllocation / totalOperation;
             var memoryLeakPerOperation = nativeLeakSize / totalOperation;
 
-            logger.WriteLine($"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
+            logger.WriteLine(
+                $"Native memory allocated per single operation: {SizeValue.FromBytes(memoryAllocatedPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}");
             logger.WriteLine($"Count of allocated object: {countOfAllocatedObject / totalOperation}");
 
             if (nativeLeakSize != 0)
             {
-                logger.WriteLine($"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, benchmarkCase.Config.CultureInfo)}");
+                logger.WriteLine(
+                    $"Native memory leak per single operation: {SizeValue.FromBytes(memoryLeakPerOperation).ToString(SizeUnit.B, null, benchmarkCase.Config.CultureInfo)}");
             }
 
             var heapInfoList = heaps.Select(h => new { Address = h.Key, h.Value.Count, types = h.Value.Values });
@@ -267,7 +269,8 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
             };
         }
 
-        private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dictionary<Address, Dictionary<Address, long>> heaps, ref Dictionary<Address, long> lastHeapAllocs, ref Address lastHeapHandle)
+        private static Dictionary<Address, long> CreateHeapCache(Address heapHandle, Dictionary<Address, Dictionary<Address, long>> heaps,
+            ref Dictionary<Address, long> lastHeapAllocs, ref Address lastHeapHandle)
         {
             Dictionary<Address, long> ret;
 
@@ -282,4 +285,4 @@ bool IsCallStackIn(StackSourceCallStackIndex index)
             return ret;
         }
     }
-}
+}
diff --git a/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/VSTestEventProcessor.cs
@@ -131,7 +131,7 @@ public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkRep
             var formatter = statistics.CreateNanosecondFormatter(cultureInfo);
 
             var builder = new StringBuilder();
-            var histogram = HistogramBuilder.Adaptive.Build(statistics.OriginalValues);
+            var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values);
             builder.AppendLine("-------------------- Histogram --------------------");
             builder.AppendLine(histogram.ToString(formatter));
             builder.AppendLine("---------------------------------------------------");

diff --git a/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs b/src/BenchmarkDotNet/Analysers/MultimodalDistributionAnalyzer.cs
@@ -20,7 +20,7 @@ protected override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report,
             if (statistics == null || statistics.N < EngineResolver.DefaultMinWorkloadIterationCount)
                 yield break;
 
-            double mValue = MValueCalculator.Calculate(statistics.OriginalValues);
+            double mValue = MValueCalculator.Calculate(statistics.Sample.Values);
             if (mValue > 4.2)
                 yield return Create("is multimodal", mValue, report, summary.GetCultureInfo());
             else if (mValue > 3.2)

diff --git a/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs b/src/BenchmarkDotNet/Analysers/OutliersAnalyser.cs
@@ -3,6 +3,7 @@
 using System.Linq;
 using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Extensions;
+using BenchmarkDotNet.Helpers;
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Reports;
 using JetBrains.Annotations;
@@ -53,7 +54,7 @@ string Format(int n, string verb)
                 return $"{n} {words} {verb}";
             }
 
-            var rangeMessages = new List<string?> { GetRangeMessage(lowerOutliers, cultureInfo), GetRangeMessage(upperOutliers, cultureInfo) };
+            var rangeMessages = new List<string?> { GetRangeMessage(lowerOutliers), GetRangeMessage(upperOutliers) };
             rangeMessages.RemoveAll(string.IsNullOrEmpty);
             string rangeMessage = rangeMessages.Any()
                 ? " (" + string.Join(", ", rangeMessages) + ")"
@@ -66,20 +67,17 @@ string Format(int n, string verb)
             return Format(actualOutliers.Length, "removed") + ", " + Format(allOutliers.Length, "detected") + rangeMessage;
         }
 
-        private static string? GetRangeMessage(double[] values, CultureInfo cultureInfo)
+        private static string? GetRangeMessage(double[] values)
         {
-            string Format(double value) => TimeInterval.FromNanoseconds(value).ToString(cultureInfo, "N2");
+            string Format(double value) => TimeInterval.FromNanoseconds(value).ToDefaultString("N2");
 
-            switch (values.Length) {
-                case 0:
-                    return null;
-                case 1:
-                    return Format(values.First());
-                case 2:
-                    return Format(values.Min()) + ", " + Format(values.Max());
-                default:
-                    return Format(values.Min()) + ".." + Format(values.Max());
-            }
+            return values.Length switch
+            {
+                0 => null,
+                1 => Format(values.First()),
+                2 => Format(values.Min()) + ", " + Format(values.Max()),
+                _ => Format(values.Min()) + ".." + Format(values.Max())
+            };
         }
     }
 }
diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementAnalyser.cs
@@ -28,17 +28,17 @@ protected override IEnumerable<Conclusion> AnalyseReport(BenchmarkReport report,
             var workloadMeasurements = entire.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual)).ToArray();
             if (workloadMeasurements.IsEmpty())
                 yield break;
-            var workload = workloadMeasurements.GetStatistics();
 
+            var workloadSample = workloadMeasurements.GetStatistics().Sample;
             var threshold = currentFrequency.Value.ToResolution().Nanoseconds / 2;
 
             var zeroMeasurement = overheadMeasurements.Any()
-                ? ZeroMeasurementHelper.CheckZeroMeasurementTwoSamples(workload.WithoutOutliers(), overheadMeasurements.GetStatistics().WithoutOutliers())
-                : ZeroMeasurementHelper.CheckZeroMeasurementOneSample(workload.WithoutOutliers(), threshold);
+                ? ZeroMeasurementHelper.AreIndistinguishable(workloadSample, overheadMeasurements.GetStatistics().Sample)
+                : ZeroMeasurementHelper.IsNegligible(workloadSample, threshold);
 
             if (zeroMeasurement)
                 yield return CreateWarning("The method duration is indistinguishable from the empty method duration",
-                                           report, false);
+                    report, false);
         }
     }
 }
diff --git a/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs b/src/BenchmarkDotNet/Analysers/ZeroMeasurementHelper.cs
@@ -1,30 +1,34 @@
+using BenchmarkDotNet.Mathematics;
+using Perfolizer;
+using Perfolizer.Mathematics.Common;
+using Perfolizer.Mathematics.GenericEstimators;
 using Perfolizer.Mathematics.SignificanceTesting;
-using Perfolizer.Mathematics.Thresholds;
+using Perfolizer.Mathematics.SignificanceTesting.MannWhitney;
+using Perfolizer.Metrology;
 
 namespace BenchmarkDotNet.Analysers
 {
-    public static class ZeroMeasurementHelper
+    internal static class ZeroMeasurementHelper
     {
-        /// <summary>
-        /// Checks distribution against Zero Measurement hypothesis in case of known threshold
-        /// </summary>
-        /// <returns>True if measurement is ZeroMeasurement</returns>
-        public static bool CheckZeroMeasurementOneSample(double[] results, double threshold)
-        {
-            if (results.Length < 3)
-                return false;
-            return !StudentTest.Instance.IsGreater(results, threshold).NullHypothesisIsRejected;
-        }
+        public static bool IsNegligible(Sample results, double threshold) => HodgesLehmannEstimator.Instance.Median(results) < threshold;
+        public static bool IsNoticeable(Sample results, double threshold) => !IsNegligible(results, threshold);
 
-        /// <summary>
-        /// Checks distribution against Zero Measurement hypothesis in case of two samples
-        /// </summary>
-        /// <returns>True if measurement is ZeroMeasurement</returns>
-        public static bool CheckZeroMeasurementTwoSamples(double[] workload, double[] overhead, Threshold? threshold = null)
+        public static bool AreIndistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) =>
+            AreIndistinguishable(new Sample(workload), new Sample(overhead), threshold);
+
+        public static bool AreIndistinguishable(Sample workload, Sample overhead, Threshold? threshold = null)
         {
-            if (workload.Length < 3 || overhead.Length < 3)
+            threshold ??= MathHelper.DefaultThreshold;
+            var tost = new SimpleEquivalenceTest(MannWhitneyTest.Instance);
+            if (workload.Size == 1 || overhead.Size == 1)
                 return false;
-            return !WelchTest.Instance.IsGreater(workload, overhead, threshold).NullHypothesisIsRejected;
+            return tost.Perform(workload, overhead, threshold, SignificanceLevel.P1E5) == ComparisonResult.Indistinguishable;
         }
+
+        public static bool AreDistinguishable(double[] workload, double[] overhead, Threshold? threshold = null) =>
+            !AreIndistinguishable(workload, overhead, threshold);
+
+        public static bool AreDistinguishable(Sample workload, Sample overhead, Threshold? threshold = null) =>
+            !AreIndistinguishable(workload, overhead, threshold);
     }
 }
diff --git a/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/ConfidenceIntervalErrorColumnAttribute.cs
@@ -7,8 +7,7 @@ namespace BenchmarkDotNet.Attributes
     [PublicAPI]
     public class ConfidenceIntervalErrorColumnAttribute : ColumnConfigBaseAttribute
     {
-        public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level = ConfidenceLevel.L999) : base(StatisticColumn.CiError(level))
-        {
-        }
+        public ConfidenceIntervalErrorColumnAttribute() : base(StatisticColumn.CiError(ConfidenceLevel.L999)) { }
+        public ConfidenceIntervalErrorColumnAttribute(ConfidenceLevel level) : base(StatisticColumn.CiError(level)) { }
     }
 }
diff --git a/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/StdErrorColumnAttribute.cs
@@ -4,10 +4,5 @@
 namespace BenchmarkDotNet.Attributes
 {
     [PublicAPI]
-    public class StdErrorColumnAttribute : ColumnConfigBaseAttribute
-    {
-        public StdErrorColumnAttribute() : base(StatisticColumn.StdErr)
-        {
-        }
-    }
+    public class StdErrorColumnAttribute() : ColumnConfigBaseAttribute(StatisticColumn.StdErr) { }
 }
diff --git a/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs b/src/BenchmarkDotNet/Attributes/Columns/WelchTTestPValueColumnAttribute.cs
@@ -1,26 +1,19 @@
 using System;
 using BenchmarkDotNet.Columns;
 using JetBrains.Annotations;
-using Perfolizer.Mathematics.SignificanceTesting;
-using Perfolizer.Mathematics.Thresholds;
+using Perfolizer.Mathematics.Common;
 
 namespace BenchmarkDotNet.Attributes
 {
     [PublicAPI]
     [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)]
     public class StatisticalTestColumnAttribute : ColumnConfigBaseAttribute
     {
-        public StatisticalTestColumnAttribute(StatisticalTestKind testKind, ThresholdUnit thresholdUnit, double value, bool showPValues = false)
-            : base(StatisticalTestColumn.Create(testKind, Threshold.Create(thresholdUnit, value), showPValues)) { }
+        public StatisticalTestColumnAttribute() : base(StatisticalTestColumn.Create("10%", null)) { }
 
-        public StatisticalTestColumnAttribute(StatisticalTestKind testKind, bool showPValues = false) : this(testKind, ThresholdUnit.Ratio, 0.1, showPValues) { }
+        public StatisticalTestColumnAttribute(string threshold) : base(StatisticalTestColumn.Create(threshold, null)) { }
 
-        public StatisticalTestColumnAttribute(bool showPValues = false) : this(StatisticalTestKind.MannWhitney, showPValues) {}
-    }
-
-    [Obsolete("Use StatisticalTestAttribute")]
-    public class WelchTTestPValueColumnAttribute : StatisticalTestColumnAttribute
-    {
-        public WelchTTestPValueColumnAttribute() : base(StatisticalTestKind.Welch) { }
+        public StatisticalTestColumnAttribute(string threshold, SignificanceLevel significanceLevel)
+            : base(StatisticalTestColumn.Create(threshold, significanceLevel)) { }
     }
 }
diff --git a/src/BenchmarkDotNet/BenchmarkDotNet.csproj b/src/BenchmarkDotNet/BenchmarkDotNet.csproj
@@ -20,7 +20,7 @@
     <PackageReference Include="Gee.External.Capstone" Version="2.3.0" />
     <PackageReference Include="Iced" Version="1.17.0" />
     <PackageReference Include="Microsoft.Diagnostics.Runtime" Version="2.2.332302" />
-    <PackageReference Include="Perfolizer" Version="[0.2.1]" />
+    <PackageReference Include="Perfolizer" Version="[0.3.16]" />
     <PackageReference Include="Microsoft.Diagnostics.Tracing.TraceEvent" Version="3.1.8" PrivateAssets="contentfiles;analyzers" />
     <PackageReference Include="Microsoft.DotNet.PlatformAbstractions" Version="3.1.6" />
     <!-- Do not update these packages, or else netcoreapp3.0 and older will no longer work -->

diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs
@@ -46,7 +46,7 @@ public IEnumerable<IColumn> GetColumns(Summary summary)
                 if (NeedToShow(summary, s => s.Percentiles.P95 > s.Mean + 3 * s.StandardDeviation))
                     yield return StatisticColumn.P95;
                 if (NeedToShow(summary, s => s.N >= 3 &&
-                                             (!s.GetConfidenceInterval(ConfidenceLevel.L99, s.N).Contains(s.Median) ||
+                                             (!s.GetConfidenceInterval(ConfidenceLevel.L99).Contains(s.Median) ||
                                               Math.Abs(s.Median - s.Mean) > s.Mean * 0.2)))
                     yield return StatisticColumn.Median;
                 if (NeedToShow(summary, s => s.StandardDeviation > 1e-9))