Throughput analysis v2.0

BitFaster.Caching.ThroughputAnalysis/BitFaster.Caching.ThroughputAnalysis.csproj

@@ -4,6 +4,7 @@
     <OutputType>Exe</OutputType>
     <TargetFramework>net6.0</TargetFramework>
     <SignAssembly>False</SignAssembly>
+    <Version>2.0.0</Version>
     <ServerGarbageCollection>true</ServerGarbageCollection>
     <ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
     <RetainVMGarbageCollection>true</RetainVMGarbageCollection>
@@ -22,7 +23,7 @@
         <NoWarn>NU1701</NoWarn>
     </PackageReference>
     <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
-    <PackageReference Include="Plotly.NET.CSharp" Version="0.11.1" />
+    <PackageReference Include="Plotly.NET" Version="4.2.0" />
     <PackageReference Include="Plotly.NET.ImageExport" Version="5.0.1" />
   </ItemGroup>
 

BitFaster.Caching.ThroughputAnalysis/ConfigFactory.cs

@@ -12,38 +12,26 @@ public class ConfigFactory
 
         public static (ThroughputBenchmarkBase, IThroughputBenchConfig, int) Create(Mode mode, int cacheSize, int maxThreads)
         {
-            int iterations = GetIterationCount(cacheSize);
             int samples = GetSampleCount(cacheSize);
             int n = cacheSize; // number of unique items for Zipf
 
             switch (mode)
             {
                 case Mode.Read:
-                    return (new ReadThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
+                    return (new ReadThroughputBenchmark(), new ZipfConfig(samples, s, n), cacheSize);
                 case Mode.ReadWrite:
                     // cache holds 10% of all items
                     cacheSize /= 10;
-                    return (new ReadThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
+                    return (new ReadThroughputBenchmark(), new ZipfConfig(samples, s, n), cacheSize);
                 case Mode.Update:
-                    return (new UpdateThroughputBenchmark(), new ZipfConfig(iterations, samples, s, n), cacheSize);
+                    return (new UpdateThroughputBenchmark(), new ZipfConfig(samples, s, n), cacheSize);
                 case Mode.Evict:
-                    return (new ReadThroughputBenchmark() { Initialize = c => EvictionInit(c) }, new EvictionConfig(iterations, samples, maxThreads), cacheSize);
+                    return (new ReadThroughputBenchmark() { Initialize = c => EvictionInit(c) }, new EvictionConfig(samples, maxThreads), cacheSize);
             }
 
             throw new InvalidOperationException();
         }
 
-        private static int GetIterationCount(int cacheSize) => cacheSize switch
-        {
-            < 500 => 400,
-            < 5_000 => 200,
-            < 10_000 => 100,
-            < 100_000 => 50,
-            < 1_000_000 => 25,
-            < 10_000_000 => 5,
-            _ => 1
-        };
-
         private static int GetSampleCount(int cacheSize) => cacheSize switch
         {
             < 5_000 => cacheSize * 4,

BitFaster.Caching.ThroughputAnalysis/Exporter.cs

@@ -9,15 +9,14 @@
 using Plotly.NET;
 using Plotly.NET.ImageExport;
 using Plotly.NET.LayoutObjects;
-using Chart = Plotly.NET.CSharp.Chart;
 
 namespace BitFaster.Caching.ThroughputAnalysis
 {
     public class Exporter
     {
         DataTable resultTable = new DataTable();
 
-        public Exporter(int maxThreads)
+        public Exporter(int minThreads, int maxThreads)
         {
             // output:
             // ThreadCount   1  2  3  4  5
@@ -26,7 +25,7 @@ public Exporter(int maxThreads)
 
             resultTable.Clear();
             resultTable.Columns.Add("ThreadCount");
-            foreach (var tc in Enumerable.Range(1, maxThreads).ToArray())
+            foreach (var tc in Enumerable.Range(minThreads, maxThreads - (minThreads-1)).ToArray())
             {
                 resultTable.Columns.Add(tc.ToString());
             }
@@ -88,10 +87,12 @@ public void ExportPlot(Mode mode, int cacheSize)
                 string name = row[0].ToString();
                 for (var i = 1; i < resultTable.Columns.Count; i++)
                 {
+                    // convert back to millions
                     rowData.Add(double.Parse(row[i].ToString()) * 1_000_000);
                 }
 
-                var chart = Chart.Line<int, double, string>(columns, rowData, Name: name, MarkerColor: MapColor(name));
+               // var chart = Chart.Line<int, double, string>(columns, rowData, Name: name, MarkerColor: MapColor(name));
+                var chart = Chart2D.Chart.Line<int, double, string>(columns, rowData, Name: name, MarkerColor: MapColor(name));
                 charts.Add(chart);
 
                 var combined = Chart.Combine(charts);

BitFaster.Caching.ThroughputAnalysis/FastZipf.cs

@@ -8,7 +8,7 @@ namespace BitFaster.Caching.ThroughputAnalysis
     /// </summary>
     public class FastZipf
     {
-        private static readonly Random srandom = new(666);
+        private static readonly Random srandom = new(42);
 
         /// <summary>
         /// Generate a zipf distribution.

BitFaster.Caching.ThroughputAnalysis/Format.cs

@@ -0,0 +1,12 @@
+namespace BitFaster.Caching.ThroughputAnalysis
+{
+    internal class Format
+    {
+        public static string Throughput(double thru)
+        {
+            string dformat = "0.00;-0.00";
+            string raw = thru.ToString(dformat);
+            return raw.PadLeft(7, ' ');
+        }
+    }
+}

BitFaster.Caching.ThroughputAnalysis/Host.cs

@@ -8,6 +8,11 @@ public class Host
     {
         public static void PrintInfo()
         {
+            var Reference = typeof(Host).Assembly;
+            var Version = Reference.GetName().Version;
+
+            Console.WriteLine($"Throughput Analysis {Version}");
+
             var hostinfo = HostEnvironmentInfo.GetCurrent();
 
             foreach (var segment in hostinfo.ToFormattedString())
@@ -35,8 +40,6 @@ public static void PrintInfo()
 
                 Console.ResetColor();
             }
-
-            Console.WriteLine();
         }
 
         public static int GetAvailableCoreCount()

BitFaster.Caching.ThroughputAnalysis/MeasurementsStatistics.cs

@@ -0,0 +1,149 @@
+using System;
+using System.Collections.Generic;
+using Perfolizer.Mathematics.Common;
+using Perfolizer.Mathematics.OutlierDetection;
+
+namespace BitFaster.Caching.ThroughputAnalysis
+{
+    // https://github.com/dotnet/BenchmarkDotNet/blob/b4ac9df9f7890ca9669e2b9c8835af35c072a453/src/BenchmarkDotNet/Mathematics/MeasurementsStatistics.cs#L13
+    internal readonly ref struct MeasurementsStatistics
+    {
+        /// <summary>
+        /// Standard error.
+        /// </summary>
+        public double StandardError { get; }
+
+        /// <summary>
+        /// Mean.
+        /// </summary>
+        public double Mean { get; }
+
+        /// <summary>
+        /// 99.9% confidence interval.
+        /// </summary>
+        public ConfidenceInterval ConfidenceInterval { get; }
+
+        private MeasurementsStatistics(double standardError, double mean, ConfidenceInterval confidenceInterval)
+        {
+            StandardError = standardError;
+            Mean = mean;
+            ConfidenceInterval = confidenceInterval;
+        }
+
+        public static MeasurementsStatistics Calculate(List<double> measurements, OutlierMode outlierMode)
+        {
+            int n = measurements.Count;
+            if (n == 0)
+                throw new InvalidOperationException("StatSummary: Sequence contains no elements");
+
+            double sum = Sum(measurements);
+            double mean = sum / n;
+
+            double variance = Variance(measurements, n, mean);
+            double standardDeviation = Math.Sqrt(variance);
+            double standardError = standardDeviation / Math.Sqrt(n);
+            var confidenceInterval = new ConfidenceInterval(mean, standardError, n);
+
+            if (outlierMode == OutlierMode.DontRemove) // most simple scenario is done without allocations! but this is not the default case
+                return new MeasurementsStatistics(standardError, mean, confidenceInterval);
+
+            measurements.Sort(); // sort in place
+
+            double q1, q3;
+
+            if (n == 1)
+                q1 = q3 = measurements[0];
+            else
+            {
+                q1 = GetQuartile(measurements, measurements.Count / 2);
+                q3 = GetQuartile(measurements, measurements.Count * 3 / 2);
+            }
+
+            double interquartileRange = q3 - q1;
+            double lowerFence = q1 - 1.5 * interquartileRange;
+            double upperFence = q3 + 1.5 * interquartileRange;
+
+            SumWithoutOutliers(outlierMode, measurements, lowerFence, upperFence, out sum, out n); // updates sum and N
+            mean = sum / n;
+
+            variance = VarianceWithoutOutliers(outlierMode, measurements, n, mean, lowerFence, upperFence);
+            standardDeviation = Math.Sqrt(variance);
+            standardError = standardDeviation / Math.Sqrt(n);
+            confidenceInterval = new ConfidenceInterval(mean, standardError, n);
+
+            return new MeasurementsStatistics(standardError, mean, confidenceInterval);
+        }
+
+        private static double Sum(List<double> measurements)
+        {
+            double sum = 0;
+            foreach (var m in measurements)
+                sum += m;
+            return sum;
+        }
+
+        private static void SumWithoutOutliers(OutlierMode outlierMode, List<double> measurements,
+            double lowerFence, double upperFence, out double sum, out int n)
+        {
+            sum = 0;
+            n = 0;
+
+            foreach (var m in measurements)
+                if (!IsOutlier(outlierMode, m, lowerFence, upperFence))
+                {
+                    sum += m;
+                    ++n;
+                }
+        }
+
+        private static double Variance(List<double> measurements, int n, double mean)
+        {
+            if (n == 1)
+                return 0;
+
+            double variance = 0;
+            foreach (var m in measurements)
+                variance += (m - mean) * (m - mean) / (n - 1);
+
+            return variance;
+        }
+
+        private static double VarianceWithoutOutliers(OutlierMode outlierMode, List<double> measurements, int n, double mean, double lowerFence, double upperFence)
+        {
+            if (n == 1)
+                return 0;
+
+            double variance = 0;
+            foreach (var m in measurements)
+                if (!IsOutlier(outlierMode, m, lowerFence, upperFence))
+                    variance += (m - mean) * (m - mean) / (n - 1);
+
+            return variance;
+        }
+
+        private static double GetQuartile(List<double> measurements, int count)
+        {
+            if (count % 2 == 0)
+                return (measurements[count / 2 - 1] + measurements[count / 2]) / 2;
+
+            return measurements[count / 2];
+        }
+
+        private static bool IsOutlier(OutlierMode outlierMode, double value, double lowerFence, double upperFence)
+        {
+            switch (outlierMode)
+            {
+                case OutlierMode.DontRemove:
+                    return false;
+                case OutlierMode.RemoveUpper:
+                    return value > upperFence;
+                case OutlierMode.RemoveLower:
+                    return value < lowerFence;
+                case OutlierMode.RemoveAll:
+                    return value < lowerFence || value > upperFence;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(outlierMode), outlierMode, null);
+            }
+        }
+    }
+}

BitFaster.Caching.ThroughputAnalysis/ParallelBenchmark.cs

@@ -1,31 +1,27 @@
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Linq;
-using System.Reflection;
-using System.Text;
 using System.Threading;
-using System.Threading.Tasks;
 
 namespace BitFaster.Caching.ThroughputAnalysis
 {
     public class ParallelBenchmark
     {
-        public static TimeSpan Run(Action<int> action, int threads)
+        public static TimeSpan Run(Action<int> action, int threadCount)
         {
-            Task[] tasks = new Task[threads];
-            ManualResetEventSlim mre = new ManualResetEventSlim();
+            Thread[] threads = new Thread[threadCount];
+            using ManualResetEventSlim signalStart = new ManualResetEventSlim();
 
-            Action<int> syncStart = taskId =>
+            Action<int> syncStart = (taskId) =>
             {
-                mre.Wait();
+                signalStart.Wait();
                 action(taskId);
             };
 
-            for (int i = 0; i < tasks.Length; i++)
+            for (int i = 0; i < threads.Length; i++)
             {
                 int index = i;
-                tasks[i] = Task.Factory.StartNew(() => syncStart(index), TaskCreationOptions.LongRunning);
+                threads[i] = new Thread(() => syncStart(index));
+                threads[i].Start();
             }
 
             // try to mitigate spam from MemoryCache
@@ -35,8 +31,12 @@ public static TimeSpan Run(Action<int> action, int threads)
             }
 
             var sw = Stopwatch.StartNew();
-            mre.Set();
-            Task.WaitAll(tasks);
+            signalStart.Set();
+            for (int i = 0; i < threads.Length; i++)
+            {
+                threads[i].Join();
+            }
+
             return sw.Elapsed;
         }
     }