From 6d632efdeb39caffea19e8d221dfff5015adc0b0 Mon Sep 17 00:00:00 2001 From: Andrey Akinshin Date: Tue, 6 Feb 2018 23:23:45 +0300 Subject: [PATCH] Histograms and multimodal distribution detection, fixes #429 --- .../Intro/IntroMultimodal.cs | 24 ++ .../MultimodalDistributionAnalyzer.cs | 37 +++ .../Columns/IterationsColumnAttribute.cs | 11 + .../Columns/MValueColumnAttribute.cs | 17 ++ .../Columns/StatisticColumn.cs | 11 +- .../Configs/DefaultConfig.cs | 1 + .../Exporters/PlainExporter.cs | 2 +- .../Extensions/StatisticsExtensions.cs | 15 +- .../Histograms/AdaptiveHistogramBuilder.cs | 162 +++++++++++++ .../Mathematics/Histograms/BinSizeRule.cs | 17 ++ .../Mathematics/Histograms/Histogram.cs | 29 +++ .../Mathematics/Histograms/HistogramBin.cs | 37 +++ .../Histograms/HistogramBuilder.cs | 13 ++ .../Histograms/HistogramExtensions.cs | 77 +++++++ .../Histograms/IHistogramBuilder.cs | 14 ++ .../Histograms/SimpleHistogramBuilder.cs | 54 +++++ .../Mathematics/MathHelper.cs | 45 ++++ .../Mathematics/RandomExtensions.cs | 14 ++ .../Mathematics/Statistics.cs | 2 + .../Running/BenchmarkRunnerCore.cs | 2 +- ...ovalTests.Exporters.Invariant.approved.txt | 12 +- ...ApprovalTests.Exporters.en-US.approved.txt | 12 +- ...ApprovalTests.Exporters.ru-RU.approved.txt | 12 +- .../Mathematics/HistogramTests.cs | 109 +++++++++ .../Histograms/AdaptiveHistogramTests.cs | 212 ++++++++++++++++++ .../Histograms/GeneralHistogramTests.cs | 23 ++ .../Histograms/HistogramTestHelper.cs | 70 ++++++ .../Mathematics/Histograms/MultimodalTests.cs | 61 +++++ .../Histograms/SimpleHistogramTests.cs | 55 +++++ 29 files changed, 1132 insertions(+), 18 deletions(-) create mode 100644 samples/BenchmarkDotNet.Samples/Intro/IntroMultimodal.cs create mode 100644 src/BenchmarkDotNet.Core/Analysers/MultimodalDistributionAnalyzer.cs create mode 100644 src/BenchmarkDotNet.Core/Attributes/Columns/IterationsColumnAttribute.cs create mode 100644 src/BenchmarkDotNet.Core/Attributes/Columns/MValueColumnAttribute.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/AdaptiveHistogramBuilder.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/BinSizeRule.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/Histogram.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBin.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBuilder.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramExtensions.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/IHistogramBuilder.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/Histograms/SimpleHistogramBuilder.cs create mode 100644 src/BenchmarkDotNet.Core/Mathematics/RandomExtensions.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/HistogramTests.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/Histograms/AdaptiveHistogramTests.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/Histograms/GeneralHistogramTests.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/Histograms/HistogramTestHelper.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/Histograms/MultimodalTests.cs create mode 100644 tests/BenchmarkDotNet.Tests/Mathematics/Histograms/SimpleHistogramTests.cs diff --git a/samples/BenchmarkDotNet.Samples/Intro/IntroMultimodal.cs b/samples/BenchmarkDotNet.Samples/Intro/IntroMultimodal.cs new file mode 100644 index 0000000000..dc1e817df8 --- /dev/null +++ b/samples/BenchmarkDotNet.Samples/Intro/IntroMultimodal.cs @@ -0,0 +1,24 @@ +using System; +using System.Threading; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Attributes.Columns; +using BenchmarkDotNet.Attributes.Jobs; +using BenchmarkDotNet.Engines; + +namespace BenchmarkDotNet.Samples.Intro +{ + [MValueColumn] + [SimpleJob(RunStrategy.Throughput, 1, 0, -1, 1, "MainJob")] + public class IntroMultimodal + { + private readonly Random rnd = new Random(42); + + private void Multimodal(int n) + => Thread.Sleep((rnd.Next(n) + 1) * 100); + + [Benchmark] public void Unimodal() => Multimodal(1); + [Benchmark] public void Bimodal() => Multimodal(2); + [Benchmark] public void Trimodal() => Multimodal(3); + [Benchmark] public void Quadrimodal() => Multimodal(4); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Analysers/MultimodalDistributionAnalyzer.cs b/src/BenchmarkDotNet.Core/Analysers/MultimodalDistributionAnalyzer.cs new file mode 100644 index 0000000000..2e796ef440 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Analysers/MultimodalDistributionAnalyzer.cs @@ -0,0 +1,37 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Reports; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Analysers +{ + public class MultimodalDistributionAnalyzer : AnalyserBase + { + public override string Id => "MultimodalDistribution"; + public static readonly IAnalyser Default = new MultimodalDistributionAnalyzer(); + + private MultimodalDistributionAnalyzer() { } + + [NotNull] + private Conclusion Create([NotNull] string kind, double mValue, [CanBeNull] BenchmarkReport report) + => CreateWarning($"It seems that the distribution {kind} (mValue = {mValue})", report); + + public override IEnumerable AnalyseReport(BenchmarkReport report, Summary summary) + { + var statistics = report.ResultStatistics; + if (statistics == null || statistics.N < 15) + yield break; + double mValue = MathHelper.CalculateMValue(statistics); + if (mValue > 4.2) + yield return Create("is multimodal", mValue, report); + else if (mValue > 3.2) + yield return Create("is bimodal", mValue, report); + else if (mValue > 2.8) + yield return Create("can have several modes", mValue, report); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Attributes/Columns/IterationsColumnAttribute.cs b/src/BenchmarkDotNet.Core/Attributes/Columns/IterationsColumnAttribute.cs new file mode 100644 index 0000000000..8c73b0e8c9 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Attributes/Columns/IterationsColumnAttribute.cs @@ -0,0 +1,11 @@ +using BenchmarkDotNet.Columns; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes.Columns +{ + [PublicAPI] + public class IterationsColumnAttribute : ColumnConfigBaseAttribute + { + public IterationsColumnAttribute() : base(StatisticColumn.Iterations) { } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Attributes/Columns/MValueColumnAttribute.cs b/src/BenchmarkDotNet.Core/Attributes/Columns/MValueColumnAttribute.cs new file mode 100644 index 0000000000..f2098ea382 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Attributes/Columns/MValueColumnAttribute.cs @@ -0,0 +1,17 @@ +using BenchmarkDotNet.Columns; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes.Columns +{ + /// + /// Prints mvalue. + /// See http://www.brendangregg.com/FrequencyTrails/modes.html + /// + [PublicAPI] + public class MValueColumnAttribute: ColumnConfigBaseAttribute + { + public MValueColumnAttribute() : base(StatisticColumn.MValue) + { + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Columns/StatisticColumn.cs b/src/BenchmarkDotNet.Core/Columns/StatisticColumn.cs index 211a79d2a2..a1428e893d 100644 --- a/src/BenchmarkDotNet.Core/Columns/StatisticColumn.cs +++ b/src/BenchmarkDotNet.Core/Columns/StatisticColumn.cs @@ -17,7 +17,7 @@ public enum Priority Percentiles, Additional } - + public static readonly IColumn Mean = new StatisticColumn("Mean", "Arithmetic mean of all measurements", s => s.Mean, Priority.Main); @@ -53,6 +53,15 @@ public enum Priority public static readonly IColumn Kurtosis = new StatisticColumn("Kurtosis", "Measure of the tailedness ( fourth standardized moment)", s => s.Kurtosis, Priority.Additional, UnitType.Dimensionless); + /// + /// See http://www.brendangregg.com/FrequencyTrails/modes.html + /// + public static readonly IColumn MValue = new StatisticColumn("MValue", "Modal value, see http://www.brendangregg.com/FrequencyTrails/modes.html", + MathHelper.CalculateMValue, Priority.Additional, UnitType.Dimensionless); + + public static readonly IColumn Iterations = new StatisticColumn("Iterations", "Number of target iterations", + s => s.N, Priority.Additional, UnitType.Dimensionless); + public static readonly IColumn P0 = CreatePercentileColumn(0, s => s.Percentiles.P0); public static readonly IColumn P25 = CreatePercentileColumn(25, s => s.Percentiles.P25); public static readonly IColumn P50 = CreatePercentileColumn(50, s => s.Percentiles.P50); diff --git a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs index 0564fb8188..6b0fa79098 100644 --- a/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet.Core/Configs/DefaultConfig.cs @@ -45,6 +45,7 @@ public IEnumerable GetAnalysers() yield return OutliersAnalyser.Default; yield return MinIterationTimeAnalyser.Default; yield return IterationSetupCleanupAnalyser.Default; + yield return MultimodalDistributionAnalyzer.Default; } public IEnumerable GetValidators() diff --git a/src/BenchmarkDotNet.Core/Exporters/PlainExporter.cs b/src/BenchmarkDotNet.Core/Exporters/PlainExporter.cs index 4d98fa42b4..cca2799174 100644 --- a/src/BenchmarkDotNet.Core/Exporters/PlainExporter.cs +++ b/src/BenchmarkDotNet.Core/Exporters/PlainExporter.cs @@ -27,7 +27,7 @@ public override void ExportToLog(Summary summary, ILogger logger) { logger.WriteLine(); logger.WriteLineHeader($"* Statistics for {mode}"); - logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStatistics().ToTimeStr()); + logger.WriteLineStatistic(runs.Where(it => it.IterationMode == mode).GetStatistics().ToTimeStr(calcHistogram: true)); } } } diff --git a/src/BenchmarkDotNet.Core/Extensions/StatisticsExtensions.cs b/src/BenchmarkDotNet.Core/Extensions/StatisticsExtensions.cs index 80b63d99ed..621c629705 100644 --- a/src/BenchmarkDotNet.Core/Extensions/StatisticsExtensions.cs +++ b/src/BenchmarkDotNet.Core/Extensions/StatisticsExtensions.cs @@ -1,6 +1,7 @@ using System.Text; using BenchmarkDotNet.Horology; using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Mathematics.Histograms; namespace BenchmarkDotNet.Extensions { @@ -24,7 +25,7 @@ public static string ToStr(this Statistics s) return builder.ToString(); } - public static string ToTimeStr(this Statistics s, TimeUnit unit = null) + public static string ToTimeStr(this Statistics s, TimeUnit unit = null, bool calcHistogram = false) { if (s == null) return NullSummaryMessage; @@ -34,12 +35,20 @@ public static string ToTimeStr(this Statistics s, TimeUnit unit = null) string errorPercent = (s.StandardError / s.Mean * 100).ToStr("0.00"); var ci = s.ConfidenceInterval; string ciMarginPercent = (ci.Margin / s.Mean * 100).ToStr("0.00"); + double mValue = MathHelper.CalculateMValue(s); builder.AppendLine($"Mean = {s.Mean.ToTimeStr(unit)}, StdErr = {s.StandardError.ToTimeStr(unit)} ({errorPercent}%); N = {s.N}, StdDev = {s.StandardDeviation.ToTimeStr(unit)}"); builder.AppendLine($"Min = {s.Min.ToTimeStr(unit)}, Q1 = {s.Q1.ToTimeStr(unit)}, Median = {s.Median.ToTimeStr(unit)}, Q3 = {s.Q3.ToTimeStr(unit)}, Max = {s.Max.ToTimeStr(unit)}"); builder.AppendLine($"IQR = {s.InterquartileRange.ToTimeStr(unit)}, LowerFence = {s.LowerFence.ToTimeStr(unit)}, UpperFence = {s.UpperFence.ToTimeStr(unit)}"); builder.AppendLine($"ConfidenceInterval = {s.ConfidenceInterval.ToTimeStr(unit)}, Margin = {ci.Margin.ToTimeStr(unit)} ({ciMarginPercent}% of Mean)"); - builder.AppendLine($"Skewness = {s.Skewness.ToStr()}, Kurtosis = {s.Kurtosis.ToStr()}"); - return builder.ToString(); + builder.AppendLine($"Skewness = {s.Skewness.ToStr()}, Kurtosis = {s.Kurtosis.ToStr()}, MValue = {mValue.ToStr()}"); + if (calcHistogram) + { + var histogram = HistogramBuilder.Adaptive.Build(s); + builder.AppendLine("-------------------- Histogram --------------------"); + builder.AppendLine(histogram.ToTimeStr()); + builder.AppendLine("---------------------------------------------------"); + } + return builder.ToString().Trim(); } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/AdaptiveHistogramBuilder.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/AdaptiveHistogramBuilder.cs new file mode 100644 index 0000000000..0e559bd6fa --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/AdaptiveHistogramBuilder.cs @@ -0,0 +1,162 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Extensions; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + public class AdaptiveHistogramBuilder : IHistogramBuilder + { + [PublicAPI, Pure] + public Histogram Build(Statistics s, BinSizeRule? rule = null) + { + double binSize = s.GetOptimalBinSize(rule); + if (Math.Abs(binSize) < 1e-9) + binSize = 1; + return BuildWithFixedBinSize(s.GetValues(), binSize); + } + + // TODO: Optimize + [PublicAPI, Pure] + public Histogram BuildWithFixedBinSize(IEnumerable values, double binSize) + { + const double eps = 1e-9; + const double margin = 0.1; + const double adaptiveFactor = 0.02; + + if (binSize < eps) + throw new ArgumentException($"binSize ({binSize.ToStr()}) should be a positive number", nameof(binSize)); + + var list = values.ToList(); + if (list.IsEmpty()) + throw new ArgumentException("Values should be non-empty", nameof(values)); + + list.Sort(); + + var points = new List { NiceFloor(list.Min() - binSize / 2), NiceCeiling(list.Max() + binSize / 2) }; + int processedPointCount = 0; + while (true) + { + int pointIndex = -1; + for (int i = processedPointCount; i < points.Count - 1; i++) + { + double adaptiveBinSize = (points[i] + points[i + 1]) / 2.0 * adaptiveFactor; + double maxSize = Math.Max(binSize * (1.0 + 2 * margin), adaptiveBinSize); + if (points[i + 1] - points[i] > maxSize) + { + pointIndex = i; + break; + } + } + + if (pointIndex == -1) + break; + + double lower = points[pointIndex]; + double upper = points[pointIndex + 1]; + + int bestIndex1 = -1; + int bestIndex2 = -1; + int bestCount = -1; + double bestDist = double.MaxValue; + + bool Inside(double x) => x > lower - eps && x < upper - eps; + + for (int i = 0; i < list.Count; i++) + if (Inside(list[i])) + { + int j = i; + while (j < list.Count && Inside(list[j]) && list[j] - list[i] < binSize) + j++; + int count = j - i; + var dist = list[j - 1] - list[i]; + if (count > bestCount || count == bestCount && dist < bestDist) + { + bestCount = count; + bestIndex1 = i; + bestIndex2 = j - 1; + bestDist = dist; + } + } + + if (bestIndex1 != -1) + { + double center = (list[bestIndex1] + list[bestIndex2]) / 2.0; + double adaptiveBinSize = Math.Max(binSize, center * adaptiveFactor); + double left = center - adaptiveBinSize / 2; + double right = Math.Min(center + adaptiveBinSize / 2, upper); + + if (left > lower + binSize * margin) + points.Insert(pointIndex + 1, NiceFloor(left)); + else if (right < upper - binSize * margin) + { + points.Insert(pointIndex + 1, NiceFloor(right)); + processedPointCount++; + } + else + processedPointCount++; + } + else + { + points.Insert(pointIndex + 1, NiceFloor(lower + binSize)); + processedPointCount++; + } + } + + var bins = new List(points.Count - 1); + int counter = 0; + for (int i = 0; i < points.Count - 1; i++) + { + var bin = new List(); + double lower = points[i]; + double upper = points[i + 1]; + + while (counter < list.Count && (list[counter] < upper || i == points.Count - 1)) + bin.Add(list[counter++]); + + bins.Add(new HistogramBin(lower, upper, bin.ToArray())); + } + + // Trim + while (bins.Any() && bins.First().IsEmpty) + bins.RemoveAt(0); + while (bins.Any() && bins.Last().IsEmpty) + bins.RemoveAt(bins.Count - 1); + + // Join small bins to neighbors + counter = 0; + double lastValue = 0; + while (counter < bins.Count) + { + if (bins[counter].HasAny) + lastValue = Math.Max(lastValue, bins[counter].Values.Last()); + double adaptiveThreshold = Math.Max(binSize / 2, lastValue * adaptiveFactor); + if (bins[counter].Gap < adaptiveThreshold) + { + double leftGap = counter > 0 ? bins[counter - 1].Gap : double.MaxValue; + double rightGap = counter < bins.Count - 1 ? bins[counter + 1].Gap : double.MaxValue; + if (leftGap < rightGap && counter > 0) + { + bins[counter - 1] = HistogramBin.Union(bins[counter - 1], bins[counter]); + bins.RemoveAt(counter); + } + else if (counter < bins.Count - 1) + { + bins[counter] = HistogramBin.Union(bins[counter], bins[counter + 1]); + bins.RemoveAt(counter + 1); + } + else + counter++; + } + else + counter++; + } + + return new Histogram(binSize, bins.ToArray()); + } + + private static double NiceFloor(double value) => Math.Floor(value * 1000) / 1000; + private static double NiceCeiling(double value) => Math.Ceiling(value * 1000) / 1000; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/BinSizeRule.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/BinSizeRule.cs new file mode 100644 index 0000000000..e665a048ba --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/BinSizeRule.cs @@ -0,0 +1,17 @@ +namespace BenchmarkDotNet.Mathematics.Histograms +{ + public enum BinSizeRule + { + FreedmanDiaconis, + + Scott, + + Scott2, + + SquareRoot, + + Sturges, + + Rice, + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/Histogram.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/Histogram.cs new file mode 100644 index 0000000000..b3bb13a851 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/Histogram.cs @@ -0,0 +1,29 @@ +using System.Linq; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + [PublicAPI] + public class Histogram + { + [PublicAPI] + public double BinSize { get; } + + [PublicAPI, NotNull] + public HistogramBin[] Bins { get; } + + internal Histogram(double binSize, [NotNull] HistogramBin[] bins) + { + BinSize = binSize; + Bins = bins; + } + + // For unit tests + [Pure, NotNull] + internal static Histogram BuildManual(double binSize, [NotNull] double[][] bins) + { + var histogramBins = bins.Select(bin => new HistogramBin(bin.Any() ? bin.Min() : 0, bin.Any() ? bin.Max() : 0, bin)).ToArray(); + return new Histogram(binSize, histogramBins); + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBin.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBin.cs new file mode 100644 index 0000000000..afb5862f8c --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBin.cs @@ -0,0 +1,37 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Horology; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + public class HistogramBin + { + public double Lower { get; } + public double Upper { get; } + public double[] Values { get; } + + public int Count => Values.Length; + public double Gap => Upper - Lower; + public bool IsEmpty => Count == 0; + public bool HasAny => Count > 0; + + public HistogramBin(double lower, double upper, double[] values) + { + Lower = lower; + Upper = upper; + Values = values; + } + + public static HistogramBin Union(HistogramBin bin1, HistogramBin bin2) => new HistogramBin( + Math.Min(bin1.Lower, bin2.Lower), + Math.Max(bin1.Upper, bin2.Upper), + bin1.Values.Union(bin2.Values).OrderBy(value => value).ToArray()); + + public override string ToString() + { + var unit = TimeUnit.GetBestTimeUnit(Values); + return $"[{Lower.ToTimeStr(unit)};{Upper.ToTimeStr(unit)}) {{{string.Join("; ", Values.Select(v => v.ToTimeStr(unit)))}}}"; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBuilder.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBuilder.cs new file mode 100644 index 0000000000..f27459357f --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramBuilder.cs @@ -0,0 +1,13 @@ +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + [PublicAPI] + public static class HistogramBuilder + { + [PublicAPI] public static readonly IHistogramBuilder Simple = new SimpleHistogramBuilder(); + [PublicAPI] public static readonly IHistogramBuilder Adaptive = new AdaptiveHistogramBuilder(); + + [PublicAPI] public static readonly IHistogramBuilder[] AllBuilders = { Simple, Adaptive }; + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramExtensions.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramExtensions.cs new file mode 100644 index 0000000000..fd50965915 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/HistogramExtensions.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Horology; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + [PublicAPI] + public static class HistogramExtensions + { + [PublicAPI, Pure] + public static int GetBinCount(this Histogram histogram) => histogram.Bins.Length; + + [PublicAPI, Pure, NotNull] + public static IEnumerable GetAllValues([NotNull] this Histogram histogram) => histogram.Bins.SelectMany(bin => bin.Values); + + [PublicAPI, Pure] + public static string ToTimeStr(this Histogram histogram, TimeUnit unit = null, char binSymbol = '@', bool full = false) + { + const string format = "0.000"; + var bins = histogram.Bins; + int binCount = histogram.Bins.Length; + if (unit == null) + unit = TimeUnit.GetBestTimeUnit(bins.SelectMany(bin => bin.Values).ToArray()); + + var lower = new string[binCount]; + var upper = new string[binCount]; + for (int i = 0; i < binCount; i++) + { + lower[i] = bins[i].Lower.ToTimeStr(unit, format: format); + upper[i] = bins[i].Upper.ToTimeStr(unit, format: format); + } + + int lowerWidth = lower.Max(it => it.Length); + int upperWidth = upper.Max(it => it.Length); + + var builder = new StringBuilder(); + for (int i = 0; i < binCount; i++) + { + string intervalStr = $"[{lower[i].PadLeft(lowerWidth)} ; {upper[i].PadLeft(upperWidth)})"; + string barStr = full + ? string.Join(", ", bins[i].Values.Select(it => it.ToTimeStr(unit, format: format))) + : new string(binSymbol, bins[i].Count); + builder.AppendLine($"{intervalStr} | {barStr}"); + } + + return builder.ToString().Trim(); + } + + [PublicAPI, Pure] + public static double GetOptimalBinSize([NotNull] this Statistics s, BinSizeRule? rule = null) + { + const BinSizeRule defaultRule = BinSizeRule.Scott2; + switch (rule ?? defaultRule) + { + case BinSizeRule.FreedmanDiaconis: + return 2 * s.InterquartileRange / Math.Pow(s.N, 1.0 / 3); + case BinSizeRule.Scott: + return 3.5 * s.StandardDeviation / Math.Pow(s.N, 1.0 / 3); + case BinSizeRule.Scott2: + return 3.5 * s.StandardDeviation / Math.Pow(s.N, 1.0 / 3) / 2.0; + case BinSizeRule.SquareRoot: + return (s.Max - s.Min) / Math.Sqrt(s.N); + case BinSizeRule.Sturges: + return (s.Max - s.Min) / (Math.Ceiling(Math.Log(s.N, 2)) + 1); + case BinSizeRule.Rice: + return (s.Max - s.Min) / (2 * Math.Pow(s.N, 1.0 / 3)); + default: + throw new ArgumentOutOfRangeException(nameof(rule), rule, null); + } + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/IHistogramBuilder.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/IHistogramBuilder.cs new file mode 100644 index 0000000000..eab289a6e1 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/IHistogramBuilder.cs @@ -0,0 +1,14 @@ +using System.Collections.Generic; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + public interface IHistogramBuilder + { + [PublicAPI, Pure, NotNull] + Histogram Build(Statistics s, BinSizeRule? rule = null); + + [PublicAPI, Pure, NotNull] + Histogram BuildWithFixedBinSize(IEnumerable values, double binSize); + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Histograms/SimpleHistogramBuilder.cs b/src/BenchmarkDotNet.Core/Mathematics/Histograms/SimpleHistogramBuilder.cs new file mode 100644 index 0000000000..2c1dc78b1e --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/Histograms/SimpleHistogramBuilder.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Extensions; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Mathematics.Histograms +{ + internal class SimpleHistogramBuilder : IHistogramBuilder + { + [PublicAPI, Pure] + public Histogram Build(Statistics s, BinSizeRule? rule = null) + { + double binSize = s.GetOptimalBinSize(rule); + return BuildWithFixedBinSize(s.GetValues(), binSize); + } + + [PublicAPI, Pure] + public Histogram BuildWithFixedBinSize(IEnumerable values, double binSize) + { + if (binSize < 1e-9) + throw new ArgumentException($"binSize ({binSize.ToStr()}) should be a positive number", nameof(binSize)); + + var list = values.ToList(); + if (list.IsEmpty()) + throw new ArgumentException("Values should be non-empty", nameof(values)); + + list.Sort(); + + int firstBin = GetBinIndex(list.First(), binSize); + int lastBin = GetBinIndex(list.Last(), binSize); + int binCount = lastBin - firstBin + 1; + + var bins = new HistogramBin[binCount]; + int counter = 0; + for (int i = 0; i < bins.Length; i++) + { + var bin = new List(); + double lower = (firstBin + i) * binSize; + double upper = (firstBin + i + 1) * binSize; + + while (counter < list.Count && (list[counter] < upper || i == bins.Length - 1)) + bin.Add(list[counter++]); + + bins[i] = new HistogramBin(lower, upper, bin.ToArray()); + } + + return new Histogram(binSize, bins); + } + + private static int GetBinIndex(double value, double binSize) => (int) Math.Floor(value / binSize); + + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/MathHelper.cs b/src/BenchmarkDotNet.Core/Mathematics/MathHelper.cs index 2d5d7fb204..7728111c4f 100644 --- a/src/BenchmarkDotNet.Core/Mathematics/MathHelper.cs +++ b/src/BenchmarkDotNet.Core/Mathematics/MathHelper.cs @@ -1,5 +1,9 @@ using System; +using System.Collections.Generic; +using System.Linq; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Mathematics.Histograms; +using JetBrains.Annotations; using static System.Math; namespace BenchmarkDotNet.Mathematics @@ -42,6 +46,7 @@ public static double Gauss(double x) 0.002141268741) * y + 0.000535310849) * y + 0.999936657524; } } + return x > 0.0 ? (z + 1.0) / 2 : (1.0 - z) / 2; } @@ -85,9 +90,49 @@ public static double InverseStudent(double p, double n) else lower = t; } + return (lower + upper) / 2; } + // See http://www.brendangregg.com/FrequencyTrails/modes.html + [PublicAPI] + public static double CalculateMValue([NotNull] Statistics originalStatistics) + { + try + { + var s = new Statistics(originalStatistics.WithoutOutliers()); + + double mValue = 0; + + double binSize = s.GetOptimalBinSize(); + if (Abs(binSize) < 1e-9) + binSize = 1; + while (true) + { + var histogram = HistogramBuilder.Adaptive.BuildWithFixedBinSize(s.GetValues(), binSize); + var x = new List(); + x.Add(0); + x.AddRange(histogram.Bins.Select(bin => bin.Count)); + x.Add(0); + + int sum = 0; + for (int i = 1; i < x.Count; i++) + sum += Abs(x[i] - x[i - 1]); + mValue = Max(mValue, sum * 1.0 / x.Max()); + + if (binSize > s.Max - s.Min) + break; + binSize *= 2.0; + } + + return mValue; + } + catch (Exception) + { + return 1; // In case of any bugs, we return 1 because it's an invalud value (mvalue is always >= 2) + } + } + public static int Clamp(int value, int min, int max) => Min(Max(value, min), max); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/RandomExtensions.cs b/src/BenchmarkDotNet.Core/Mathematics/RandomExtensions.cs new file mode 100644 index 0000000000..8ca6805120 --- /dev/null +++ b/src/BenchmarkDotNet.Core/Mathematics/RandomExtensions.cs @@ -0,0 +1,14 @@ +using System; + +namespace BenchmarkDotNet.Mathematics +{ + internal static class RandomExtensions + { + // See https://stackoverflow.com/questions/218060/random-gaussian-variables + public static double NextGaussian(this Random random, double mean = 0, double stdDev = 1) + { + double stdDevFactor = Math.Sqrt(-2.0 * Math.Log(random.NextDouble())) * Math.Sin(2.0 * Math.PI * random.NextDouble()); + return mean + stdDev * stdDevFactor; + } + } +} \ No newline at end of file diff --git a/src/BenchmarkDotNet.Core/Mathematics/Statistics.cs b/src/BenchmarkDotNet.Core/Mathematics/Statistics.cs index 62c70bf626..a6b937fb79 100644 --- a/src/BenchmarkDotNet.Core/Mathematics/Statistics.cs +++ b/src/BenchmarkDotNet.Core/Mathematics/Statistics.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Runtime.Serialization; using BenchmarkDotNet.Extensions; namespace BenchmarkDotNet.Mathematics @@ -80,6 +81,7 @@ public Statistics(IEnumerable values) public ConfidenceInterval GetConfidenceInterval(ConfidenceLevel level, int n) => new ConfidenceInterval(Mean, StandardError, n, level); public bool IsOutlier(double value) => value < LowerFence || value > UpperFence; public double[] WithoutOutliers() => list.Where(value => !IsOutlier(value)).ToArray(); + public IEnumerable GetValues() => list; public double CalcCentralMoment(int k) => list.Average(x => (x - Mean).Pow(k)); diff --git a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs index 79b31055ed..4ea8d9a962 100644 --- a/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs +++ b/src/BenchmarkDotNet.Core/Running/BenchmarkRunnerCore.cs @@ -154,7 +154,7 @@ public static Summary Run(BenchmarkRunInfo benchmarkRunInfo, ILogger logger, str if (resultRuns.IsEmpty()) logger.WriteLineError("There are not any results runs"); else - logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr()); + logger.WriteLineStatistic(resultRuns.GetStatistics().ToTimeStr(calcHistogram: true)); logger.WriteLine(); } diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt index 89ad88185f..08b6e73b06 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.Invariant.approved.txt @@ -418,8 +418,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(LaunchCount=3, TargetCount=100, WarmupCount=15) *** * Raw * Result 1: 1 op, 1 ns, 1.0000 ns/op @@ -429,8 +431,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- ############################################ XmlExporter-brief ############################################ diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt index 89ad88185f..08b6e73b06 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.en-US.approved.txt @@ -418,8 +418,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(LaunchCount=3, TargetCount=100, WarmupCount=15) *** * Raw * Result 1: 1 op, 1 ns, 1.0000 ns/op @@ -429,8 +431,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- ############################################ XmlExporter-brief ############################################ diff --git a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt index 89ad88185f..08b6e73b06 100644 --- a/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt +++ b/tests/BenchmarkDotNet.Tests/Exporters/ApprovedFiles/CommonExporterApprovalTests.Exporters.ru-RU.approved.txt @@ -418,8 +418,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- *** MockBenchmarkClass.Bar: LongRun(LaunchCount=3, TargetCount=100, WarmupCount=15) *** * Raw * Result 1: 1 op, 1 ns, 1.0000 ns/op @@ -429,8 +431,10 @@ Mean = 1.0000 ns, StdErr = 0.0000 ns (0.00%); N = 1, StdDev = 0.0000 ns Min = 1.0000 ns, Q1 = 1.0000 ns, Median = 1.0000 ns, Q3 = 1.0000 ns, Max = 1.0000 ns IQR = 0.0000 ns, LowerFence = 1.0000 ns, UpperFence = 1.0000 ns ConfidenceInterval = [NaN ns; NaN ns] (CI 99.9%), Margin = NaN ns (NaN% of Mean) -Skewness = NaN, Kurtosis = NaN - +Skewness = NaN, Kurtosis = NaN, MValue = 2 +-------------------- Histogram -------------------- +[0.500 ns ; 1.500 ns) | @ +--------------------------------------------------- ############################################ XmlExporter-brief ############################################ diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/HistogramTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/HistogramTests.cs new file mode 100644 index 0000000000..8a09f0bf59 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/HistogramTests.cs @@ -0,0 +1,109 @@ +using System; +using System.Linq; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Mathematics.Histograms; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Mathematics +{ + public class HistogramTests + { + private readonly ITestOutputHelper output; + + public HistogramTests(ITestOutputHelper output) => this.output = output; + + [Fact] + public void EmptyListTest() + { + foreach (var builder in HistogramBuilder.AllBuilders) + Assert.Throws(() => builder.BuildWithFixedBinSize(Array.Empty(), 1)); + } + + [Fact] + public void NegativeBinSizeTest() + { + foreach (var builder in HistogramBuilder.AllBuilders) + Assert.Throws(() => builder.BuildWithFixedBinSize(new double[] { 1, 2 }, -3)); + } + + [Fact] + public void SimpeHistogramTest1() + { + DoSimpleHistogramTest(new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, 1, + new[] + { + new[] { 1.0 }, + new[] { 2.0 }, + new[] { 3.0 }, + new[] { 4.0 }, + new[] { 5.0 } + }); + } + + [Fact] + public void SimpeHistogramTest2() + { + DoSimpleHistogramTest(new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, 2.5, + new[] + { + new[] { 1.0, 2.0 }, + new[] { 3.0, 4.0 }, + new[] { 5.0 } + }); + } + + [Fact] + public void SimpeHistogramTest3() + { + DoSimpleHistogramTest(new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 2.7 }, 2.0, + new[] + { + new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5 }, + new[] { 2.7 } + }); + } + + private void DoSimpleHistogramTest(double[] values, double binSize, double[][] bins) + { + var expectedHistogram = Histogram.BuildManual(binSize, bins); + var actualHistogram = HistogramBuilder.Simple.BuildWithFixedBinSize(values, binSize); + PrintHistogram("Expected", expectedHistogram); + PrintHistogram("Actual", actualHistogram); + + Assert.Equal(bins.Length, actualHistogram.Bins.Length); + for (int i = 0; i < actualHistogram.Bins.Length; i++) + Assert.Equal(bins[i], actualHistogram.Bins[i].Values); + } + + private void PrintHistogram(string title, Histogram histogram) + { + output.WriteLine($"=== {title}:Short ==="); + output.WriteLine(histogram.ToTimeStr()); + output.WriteLine($"=== {title}:Full ==="); + output.WriteLine(histogram.ToTimeStr(full: true)); + } + + [Theory] + [InlineData(new[] { 1.0, 2.0, 3.0 })] + [InlineData(new[] { 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0 })] + [InlineData(new[] { 0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9 })] + [InlineData(new[] { 1.0, 1, 1, 2, 2, 2, 3, 3, 3 })] + [InlineData(new[] { 100.0, 100, 100, 200, 200, 200, 300, 300, 300 })] + [InlineData(new[] { 1.0, 1.01, 1.02, 1.03, 1.03, 1.04, 1.05, 1.01, 1.02, 1.03, 1.02 })] + private void BinSizeTest(double[] values) + { + var rules = Enum.GetValues(typeof(BinSizeRule)).Cast(); + var s = new Statistics(values); + foreach (var rule in rules) + { + var histogram = HistogramBuilder.Simple.Build(s, rule); + output.WriteLine($"!!!!! Rule = {rule}, BinSize = {histogram.BinSize.ToTimeStr()} !!!!!"); + output.WriteLine(histogram.ToTimeStr()); + output.WriteLine(""); + output.WriteLine(""); + } + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/AdaptiveHistogramTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/AdaptiveHistogramTests.cs new file mode 100644 index 0000000000..5ee3925c05 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/AdaptiveHistogramTests.cs @@ -0,0 +1,212 @@ +using System; +using BenchmarkDotNet.Mathematics.Histograms; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Mathematics.Histograms +{ + public class AdaptiveHistogramTests + { + private readonly ITestOutputHelper output; + + public AdaptiveHistogramTests(ITestOutputHelper output) => this.output = output; + + [Fact] + public void TrivialTest1() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, + 1, + new[] + { + new[] { 1.0 }, + new[] { 2.0 }, + new[] { 3.0 }, + new[] { 4.0 }, + new[] { 5.0 } + }); + } + + [Fact] + public void TrivialTest2() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, + 2.5, + new[] + { + new[] { 1.0, 2.0, 3.0 }, + new[] { 4.0, 5.0 } + }); + } + + [Fact] + public void TrivialTest3() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 2.7 }, + 2.0, + new[] + { + new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 2.7 } + }); + } + + [Fact] + public void AutoSyntheticTestTrimodal() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] { 1.0, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3 }, + new[] + { + new[] { 1.0, 1, 1, 1 }, + Array.Empty(), + new[] { 2.0, 2, 2, 2 }, + Array.Empty(), + new[] { 3.0, 3, 3, 3 } + }); + } + + [Fact] + public void AutoTestUnimodal1() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 54.1166, 54.4516, 53.1706, 54.5196, 51.9886, 54.4686, 52.9636, 52.9576, 54.3516, 53.7846, 53.6686, 51.8006, 53.5406, 53.6156, 53.4546 + }, + new[] + { + true, + true + }); + } + + [Fact] + public void AutoTestUnimodal2() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 33.5455, 32.2774, 33.6649, 35.3296, 31.6737, 31.6836, 31.7396, 31.7001, 33.6881, 30.6352, 30.6761, 32.5385, 31.1269, 33.0186, 31.7488, + 30.9851, 30.7059, 31.5718, 32.9712, 31.3161, 30.5636, 30.6786, 31.1183, 30.9629, 31.4356, 33.9919, 31.4916, 30.5728, 31.6186, 30.8011, + 30.8926, 30.2854, 30.2282, 30.5786, 30.6909, 30.2862, 30.8433, 31.8990, 32.1198, 31.5409, 31.7843, 30.4518, 34.7756, 31.1336, 32.5714, + 32.2337 + }, + new[] + { + true, + true, + true, + true, + true + }); + } + + [Fact] + public void AutoTestUnimodal3() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 24.8822, 24.8288, 24.8050, 24.7990, 24.8510, 24.8993, 24.8376, 24.8345, 24.9441, 24.9758, 25.0026, 24.8308 + }, + new[] + { + true + }); + } + + [Fact] + public void AutoTestBimodal() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 53.5559, 53.6549, 101.8009, 53.6549, 54.3409, 100.9439, 103.3839, 54.1319, 103.8739, 54.4979, 53.9889, 104.2749, 53.1299, 54.2599, 54.4389, + 101.7749, 52.2869, 103.0219, 103.0079, 54.9709, 54.5199, 52.8009, 104.3999, 104.7339, 102.5139, 54.1979, 102.3789, 54.9309, 103.6629, + 104.1529, 104.1629, 53.8849, 52.2119, 53.7939, 103.8419, 101.2259, 54.8619, 101.3639, 100.7259, 104.2899, 52.1469, 54.8179, 50.0849, + 52.8589, 103.5959, 105.1049, 54.9159, 52.1189, 103.0839, 53.2739, 103.0819, 54.6049, 53.7459, 102.8169, 103.8609, 53.8309, 53.2619, + 103.9679, 101.7129, 50.1929, 54.2469, 54.4669, 54.6009, 54.3319, 101.7119, 53.5829, 52.2749, 53.7199, 104.1699, 54.7329, 100.8459, 53.7399, + 53.1539, 104.8099, 53.2049, 54.2959, 50.8879, 104.5729, 51.0259, 103.6519, 103.6909, 54.5209, 51.7519, 101.5849, 54.9299, 54.1139, 53.4199, + 55.0609, 104.4749, 104.0589, 53.1139, 54.3649, 103.9969, 104.9919, 101.0799, 53.5399, 101.3219, 101.7309, 51.0129, 104.1249, + }, + new[] + { + true, false, false, false, false, + true + }); + } + + [Fact] + public void AutoTestTrimodal() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 53.8268, 54.9398, 103.4098, 53.5118, 52.1718, 154.0758, 103.4258, 54.5178, 153.9198, 54.7608, 52.1608, 103.1478, 54.2528, 104.9618, 54.9458, + 104.8898, 53.9168, 153.1668, 104.9088, 105.0408, 53.7018, 53.6178, 153.7938, 154.3638, 104.6428, 55.0258, 152.6418, 54.1728, 152.4818, + 154.9568, 102.4368, 53.8458, 50.5228, 54.9138, 103.9068, 154.3708, 53.8648, 103.1468, 154.1558, 153.5758, 103.9948, 53.2298, 54.5768, + 53.3598, 153.4818, 102.8688, 54.3768, 54.3298, 153.0418, 54.0498, 101.2148, 54.6768, 53.2578, 104.8848, 101.6078, 53.6078, 104.3758, + 153.3428, 105.0318, 54.4548, 53.1688, 105.0628, 51.7798, 52.6878, 152.5438, 54.8058, 103.9618, 53.2448, 153.4528, 103.6548, 103.3058, + 104.7568, 52.8858, 153.3038, 51.9488, 54.3588, 52.6368, 103.5778, 53.1848, 153.8708, 154.9128, 103.9158, 104.4708, 104.9618, 53.2018, + 103.7408, 51.8718, 104.8708, 153.6998, 103.3718, 103.2448, 54.1768, 154.5538, 101.7978, 153.8588, 51.9978, 152.5858, 154.6038, 53.6288, + 153.7528, + }, + new[] + { + true, false, false, + true, false, false, + true + }); + } + + [Fact] + public void AutoTestQuadrimodal1() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 54.4140, 54.8640, 151.6690, 54.7780, 104.6170, 154.8150, 150.3550, 51.7410, 203.5600, 52.9680, 102.1660, 155.1540, 101.4900, 105.0790, + 104.8150, 154.0340, 54.5340, 205.1420, 155.0590, 103.1720, 50.8400, 54.5400, 150.3000, 201.8370, 153.4970, 53.3890, 153.5630, 53.6010, + 202.4930, 154.6960, 153.3370, 54.6520, 53.6120, 103.7030, 151.6260, 203.8850, 50.6300, 155.1610, 203.0760, 153.8910, 104.9950, 54.6320, + 54.6240, 54.2760, 202.4930, 154.8500, 50.7690, 51.9110, 203.8590, 102.5350, 154.8540, 50.1520, 54.6250, 152.8100, 154.8730, 55.0540, + 101.2380, 151.5090, 154.6420, 55.0550, 54.5840, 104.1540, 100.6920, 53.9030, 153.8870, 53.5660, 101.3580, 51.4430, 204.0370, 101.4220, + 153.5490, 102.1340, 54.6950, 204.0110, 53.1390, 53.1830, 52.9660, 153.8270, 52.4380, 202.6190, 202.6890, 100.2270, 105.0480, 150.1560, + 53.5010, 102.9840, 53.9720, 104.0280, 204.0530, 153.1220, 103.0630, 103.7960, 203.6950, 154.0190, 151.5000, 54.3830, 203.1370, 202.2610, + 103.6030, 152.8300 + }, + new[] + { + true, false, + true, false, + true, false, + true + }); + } + + [Fact] + public void AutoTestQuadrimodal2() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Adaptive, + new[] + { + 52.9868, 53.7488, 152.8768, 50.8788, 102.7788, 155.1378, 153.6468, 52.8868, 201.8688, 53.8768, 103.4568, 153.5178, 102.9078, 104.0128, + 104.1018, 153.3768, 51.0948, 203.6908, 153.9508, 103.0448, 54.5758, 53.3828, 152.4038, 204.6468, 153.3098, 53.0028, 153.3958, 54.5438, + 203.6268, 151.8578, 154.7498, 53.4388, 54.9368, 103.9388, 154.9488, 204.9338, 54.5718, 153.8728, 205.0888, 154.4578, 103.7568, 53.7358, + 52.5138, 52.6698, 201.0788, 153.3238, 54.7338, 55.0328, 204.6378, 104.8608, 151.4808, 51.0258, 54.1618, 154.9518, 151.5528, 54.7568, + 104.2438, 154.8118, 152.9708, 53.6878, 54.5128, 102.5938, 105.0188, 51.5798, 153.3078, 53.1978, 103.2718, 53.0328, 203.1028, 104.9488, + 153.6708, 105.1028, 54.4878, 202.5518, 50.2618, 51.9998, 54.6478, 153.9048, 53.8258, 203.7968, 204.0508, 102.5758, 103.1168, 153.1998, + 53.0008, 103.1648, 52.8238, 104.6748, 204.2478, 154.6658, 103.0448, 103.4738, 203.2368, 154.0128, 151.6948, 54.6178, 200.8398, 203.0058, + 104.8248, 153.7678 + }, + new[] + { + true, false, + true, false, + true, false, + true + }); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/GeneralHistogramTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/GeneralHistogramTests.cs new file mode 100644 index 0000000000..a81c8c2874 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/GeneralHistogramTests.cs @@ -0,0 +1,23 @@ +using System; +using BenchmarkDotNet.Mathematics.Histograms; +using Xunit; + +namespace BenchmarkDotNet.Tests.Mathematics.Histograms +{ + public class GeneralHistogramTests + { + [Fact] + public void EmptyListTest() + { + foreach (var builder in HistogramBuilder.AllBuilders) + Assert.Throws(() => builder.BuildWithFixedBinSize(Array.Empty(), 1)); + } + + [Fact] + public void NegativeBinSizeTest() + { + foreach (var builder in HistogramBuilder.AllBuilders) + Assert.Throws(() => builder.BuildWithFixedBinSize(new double[] { 1, 2 }, -3)); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/HistogramTestHelper.cs b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/HistogramTestHelper.cs new file mode 100644 index 0000000000..9a03725c92 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/HistogramTestHelper.cs @@ -0,0 +1,70 @@ +using System.Linq; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Mathematics.Histograms; +using JetBrains.Annotations; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Mathematics.Histograms +{ + public static class HistogramTestHelper + { + [AssertionMethod] + public static void DoHistogramTest([NotNull] ITestOutputHelper output, [NotNull] IHistogramBuilder builder, + [NotNull] double[] values, [NotNull] double[][] bins) + { + var actualHistogram = builder.Build(new Statistics(values)); + Check(output, bins, actualHistogram); + } + + [AssertionMethod] + public static void DoHistogramTest([NotNull] ITestOutputHelper output, [NotNull] IHistogramBuilder builder, + [NotNull] double[] values, double binSize, [NotNull] double[][] bins) + { + var actualHistogram = builder.BuildWithFixedBinSize(values, binSize); + Check(output, bins, actualHistogram); + } + + [AssertionMethod] + public static void DoHistogramTest([NotNull] ITestOutputHelper output, [NotNull] IHistogramBuilder builder, + [NotNull] double[] values, [NotNull] bool[] states) + { + var actualHistogram = builder.Build(new Statistics(values)); + Check(output, states, actualHistogram); + } + + [AssertionMethod] + private static void Check([NotNull] ITestOutputHelper output, [NotNull] double[][] expectedBins, Histogram actualHistogram) + { + var expectedHistogram = Histogram.BuildManual(0, expectedBins); + output.Print("Expected", expectedHistogram); + output.Print("Actual", actualHistogram); + + Assert.Equal(expectedBins.Length, actualHistogram.Bins.Length); + for (int i = 0; i < actualHistogram.Bins.Length; i++) + Assert.Equal(expectedBins[i], actualHistogram.Bins[i].Values); + } + + [AssertionMethod] + private static void Check([NotNull] ITestOutputHelper output, [NotNull] bool[] expectedStates, Histogram actualHistogram) + { + output.Print("Actual", actualHistogram); + + Assert.Equal(expectedStates.Length, actualHistogram.Bins.Length); + for (int i = 0; i < actualHistogram.Bins.Length; i++) + Assert.Equal(expectedStates[i], actualHistogram.Bins[i].HasAny); + } + + public static void Print([NotNull] this ITestOutputHelper output, [NotNull] string title, [NotNull] Histogram histogram) + { + var s = new Statistics(histogram.GetAllValues()); + double mValue = MathHelper.CalculateMValue(s); + output.WriteLine($"=== {title}:Short (BinSize={histogram.BinSize.ToTimeStr()}, mValue={mValue.ToStr()}) ==="); + output.WriteLine(histogram.ToTimeStr()); + output.WriteLine($"=== {title}:Full (BinSize={histogram.BinSize.ToTimeStr()}, mValue={mValue.ToStr()}) ==="); + output.WriteLine(histogram.ToTimeStr(full: true)); + output.WriteLine("OUTLIERS: ", string.Join(", ", s.Outliers.Select(it => it.ToTimeStr()))); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/MultimodalTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/MultimodalTests.cs new file mode 100644 index 0000000000..937b3f73e3 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/MultimodalTests.cs @@ -0,0 +1,61 @@ +using System; +using BenchmarkDotNet.Mathematics; +using BenchmarkDotNet.Mathematics.Histograms; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Mathematics.Histograms +{ + public class MultimodalTests + { + private readonly ITestOutputHelper output; + + public MultimodalTests(ITestOutputHelper output) => this.output = output; + + [Theory] + [InlineData(new double[] { 1, 1, 1, 1, 1, 1 }, 2)] + [InlineData(new double[] { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2 }, 4)] + [InlineData(new double[] { 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3 }, 6)] + [InlineData(new double[] { 1, 2, 3, 3, 3, 4, 5, 10, 11, 11, 11, 12, 40, 41, 41, 41, 42 }, 2.8333)] + public void MValueTest(double[] values, double expectedMValue) + { + var s = new Statistics(values); + var histogram = HistogramBuilder.Adaptive.Build(s); + output.Print("Distribution", histogram); + + double acutalMValue = MathHelper.CalculateMValue(s); + Assert.Equal(expectedMValue, acutalMValue, 4); + } + + [Fact] + public void RandomTest() + { + var random = new Random(42); + double maxMValue = 0; + int maxMValueN = 0; + for (int n = 1; n <= 300; n++) + { + var values = new double[n]; + for (int i = 0; i < n; i++) + values[i] = random.NextGaussian(50, 3); + + var s = new Statistics(values); + var histogram = HistogramBuilder.Adaptive.Build(s); + output.Print($"n={n}", histogram); + output.WriteLine("-------------------------------------------------------------------------"); + output.WriteLine("-------------------------------------------------------------------------"); + output.WriteLine("-------------------------------------------------------------------------"); + + double mValue = MathHelper.CalculateMValue(s); + Assert.True(mValue >= 2 - 1e-9); + + if (mValue > maxMValue) + { + maxMValue = mValue; + maxMValueN = n; + } + } + output.WriteLine($"maxMValue = {maxMValue} (N = {maxMValueN})"); + } + } +} \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/SimpleHistogramTests.cs b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/SimpleHistogramTests.cs new file mode 100644 index 0000000000..cf806dc214 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Mathematics/Histograms/SimpleHistogramTests.cs @@ -0,0 +1,55 @@ +using BenchmarkDotNet.Mathematics.Histograms; +using Xunit; +using Xunit.Abstractions; + +namespace BenchmarkDotNet.Tests.Mathematics.Histograms +{ + public class SimpleHistogramTests + { + private readonly ITestOutputHelper output; + + public SimpleHistogramTests(ITestOutputHelper output) => this.output = output; + + [Fact] + public void TrivialTest1() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Simple, + new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, + 1, + new[] + { + new[] { 1.0 }, + new[] { 2.0 }, + new[] { 3.0 }, + new[] { 4.0 }, + new[] { 5.0 } + }); + } + + [Fact] + public void TrivialTest2() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Simple, + new[] { 1.0, 2.0, 3.0, 4.0, 5.0 }, + 2.5, + new[] + { + new[] { 1.0, 2.0 }, + new[] { 3.0, 4.0 }, + new[] { 5.0 } + }); + } + + [Fact] + public void TrivialTest3() + { + HistogramTestHelper.DoHistogramTest(output, HistogramBuilder.Simple, + new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5, 2.7 }, 2.0, + new[] + { + new[] { 1.0, 1.1, 1.2, 1.3, 1.4, 1.5 }, + new[] { 2.7 } + }); + } + } +} \ No newline at end of file