Changed diagnosers flow, reduced heap allocations in Engine to 0

samples/BenchmarkDotNet.Samples/Intro/IntroGcMode.cs

@@ -14,10 +14,13 @@ private class Config : ManualConfig
         {
             public Config()
             {
-                Add(Job.Dry.WithGcServer(true).WithGcForce(true).WithId("ServerForce"));
-                Add(Job.Dry.WithGcServer(true).WithGcForce(false).WithId("Server"));
-                Add(Job.Dry.WithGcServer(false).WithGcForce(true).WithId("Workstation"));
-                Add(Job.Dry.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce"));
+                Add(Job.MediumRun.WithGcServer(true).WithGcForce(true).WithId("ServerForce"));
+                Add(Job.MediumRun.WithGcServer(true).WithGcForce(false).WithId("Server"));
+                Add(Job.MediumRun.WithGcServer(false).WithGcForce(true).WithId("Workstation"));
+                Add(Job.MediumRun.WithGcServer(false).WithGcForce(false).WithId("WorkstationForce"));
+#if !CORE
+                Add(new Diagnostics.Windows.MemoryDiagnoser());
+#endif
             }
         }
 

samples/BenchmarkDotNet.Samples/JIT/Jit_Inlining.cs

@@ -5,6 +5,9 @@ namespace BenchmarkDotNet.Samples.JIT
 {
     // See http://en.wikipedia.org/wiki/Inline_expansion
     // See http://aakinshin.net/en/blog/dotnet/inlining-and-starg/
+#if !CORE
+    [Diagnostics.Windows.Configs.InliningDiagnoserConfig]
+#endif
     [LegacyJitX86Job, LegacyJitX64Job, RyuJitX64Job]
     public class Jit_Inlining
     {

samples/BenchmarkDotNet.Samples/Program.cs

@@ -1,7 +1,5 @@
-using System;
-using System.Reflection;
+using System.Reflection;
 using BenchmarkDotNet.Running;
-using BenchmarkDotNet.Attributes;
 
 namespace BenchmarkDotNet.Samples
 {

src/BenchmarkDotNet.Core/Code/CodeGenerator.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Core.Helpers;
+using BenchmarkDotNet.Engines;
 using BenchmarkDotNet.Environments;
 using BenchmarkDotNet.Extensions;
 using BenchmarkDotNet.Helpers;
@@ -37,6 +38,7 @@ internal static string Generate(Benchmark benchmark)
                 Replace("$JobSetDefinition$", GetJobsSetDefinition(benchmark)).
                 Replace("$ParamsContent$", GetParamsContent(benchmark)).
                 Replace("$ExtraAttribute$", GetExtraAttributes(benchmark.Target)).
+                Replace("$EngineFactoryType$", typeof(EngineFactory).GetCorrectTypeName()). // todo: get it from Job's settings
                 ToString();
 
             text = Unroll(text, benchmark.Job.Run.UnrollFactor.Resolve(EnvResolver.Instance));

src/BenchmarkDotNet.Core/Diagnosers/CompositeDiagnoser.cs

@@ -4,6 +4,7 @@
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
 using BenchmarkDotNet.Reports;
+using BenchmarkDotNet.Extensions;
 
 namespace BenchmarkDotNet.Diagnosers
 {
@@ -16,35 +17,19 @@ public CompositeDiagnoser(params IDiagnoser[] diagnosers)
             this.diagnosers = diagnosers;
         }
 
-        public void Start(Benchmark benchmark)
-        {
-            foreach (var diagnoser in diagnosers)
-                diagnoser.Start(benchmark);
-        }
+        public IColumnProvider GetColumnProvider() 
+            => new CompositeColumnProvider(diagnosers.Select(d => d.GetColumnProvider()).ToArray());
 
-        public void Stop(Benchmark benchmark, BenchmarkReport report)
-        {
-            foreach (var diagnoser in diagnosers)
-                diagnoser.Stop(benchmark, report);
-        }
+        public void BeforeAnythingElse(Process process, Benchmark benchmark) 
+            => diagnosers.ForEach(diagnoser => diagnoser.BeforeAnythingElse(process, benchmark));
 
-        public void ProcessStarted(Process process)
-        {
-            foreach (var diagnoser in diagnosers)
-                diagnoser.ProcessStarted(process);
-        }
+        public void AfterSetup(Process process, Benchmark benchmark) 
+            => diagnosers.ForEach(diagnoser => diagnoser.AfterSetup(process, benchmark));
 
-        public void AfterBenchmarkHasRun(Benchmark benchmark, Process process)
-        {
-            foreach (var diagnoser in diagnosers)
-                diagnoser.AfterBenchmarkHasRun(benchmark, process);
-        }
+        public void BeforeCleanup() => diagnosers.ForEach(diagnoser => diagnoser.BeforeCleanup());
 
-        public void ProcessStopped(Process process)
-        {
-            foreach (var diagnoser in diagnosers)
-                diagnoser.ProcessStopped(process);
-        }
+        public void ProcessResults(Benchmark benchmark, BenchmarkReport report)
+            => diagnosers.ForEach(diagnoser => diagnoser.ProcessResults(benchmark, report));
 
         public void DisplayResults(ILogger logger)
         {
@@ -57,7 +42,5 @@ public void DisplayResults(ILogger logger)
                 logger.WriteLine();
             }
         }
-
-        public IColumnProvider GetColumnProvider() => new CompositeColumnProvider(diagnosers.Select(d => d.GetColumnProvider()).ToArray());
     }
 }

src/BenchmarkDotNet.Core/Diagnosers/IDiagnoser.cs

@@ -6,28 +6,27 @@
 
 namespace BenchmarkDotNet.Diagnosers
 {
-    /// The events are guaranteed to happen in the following sequence:
-    /// Start                  // When the Benchmark run is started and most importantly BEFORE the process has been launched
-    /// ProcessStarted         // After the Process (in a "Diagnostic" run) has been launched
-    /// AfterBenchmarkHasRun   // After a "Warmup" iteration of the Benchmark has run, i.e. we know the [Benchmark] method has been 
-    ///                        // executed and JITted, this is important if the Diagnoser needs to know when it can do a Memory Dump.
-    /// ProcessStopped         // Once the Process (in a "Diagnostic" run) has stopped/completed
-    /// Stop                   // At the end, when the entire Benchmark run has complete
-    /// DisplayResults         // When the results/output should be displayed
     public interface IDiagnoser
     {
-        void Start(Benchmark benchmark);
+        IColumnProvider GetColumnProvider();
 
-        void Stop(Benchmark benchmark, BenchmarkReport report);
+        /// <summary>
+        /// before jitting, warmup
+        /// </summary>
+        void BeforeAnythingElse(Process process, Benchmark benchmark);
 
-        void ProcessStarted(Process process);
+        /// <summary>
+        /// after setup, before run
+        /// </summary>
+        void AfterSetup(Process process, Benchmark benchmark);
 
-        void AfterBenchmarkHasRun(Benchmark benchmark, Process process);
+        /// <summary>
+        /// after run, before cleanup
+        /// </summary>
+        void BeforeCleanup();
 
-        void ProcessStopped(Process process);
+        void ProcessResults(Benchmark benchmark, BenchmarkReport report);
 
         void DisplayResults(ILogger logger);
-
-        IColumnProvider GetColumnProvider();
     }
 }

src/BenchmarkDotNet.Core/Engines/Engine.cs

@@ -1,10 +1,8 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using BenchmarkDotNet.Characteristics;
 using BenchmarkDotNet.Horology;
 using BenchmarkDotNet.Jobs;
-using BenchmarkDotNet.Mathematics;
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 using JetBrains.Annotations;
@@ -17,82 +15,94 @@ public class Engine : IEngine
         public const int MinInvokeCount = 4;
         public static readonly TimeInterval MinIterationTime = 200 * TimeInterval.Millisecond;
 
-        public Job TargetJob { get; set; } = Job.Default;
-        public long OperationsPerInvoke { get; set; } = 1;
-        public Action SetupAction { get; set; } = null;
-        public Action CleanupAction { get; set; } = null;
         public Action<long> MainAction { get; }
         public Action<long> IdleAction { get; }
+        public Job TargetJob { get; }
+        public long OperationsPerInvoke { get; }
+        public Action SetupAction { get; }
+        public Action CleanupAction { get; }
+        public bool IsDiagnoserAttached { get; }
+        public IResolver Resolver { get; }
+
+        private IClock Clock { get; }
+        private bool ForceAllocations { get; }
+        private int UnrollFactor { get; }
+        private RunStrategy Strategy { get; }
+        private bool EvaluateOverhead { get; }
+        private int InvocationCount { get; }
 
         private readonly EnginePilotStage pilotStage;
         private readonly EngineWarmupStage warmupStage;
         private readonly EngineTargetStage targetStage;
+        private bool isJitted, isPreAllocated;
 
-        public IResolver Resolver { get; }
-
-        public Engine([NotNull] Action<long> idleAction, [NotNull] Action<long> mainAction)
+        internal Engine(Action<long> idleAction, Action<long> mainAction, Job targetJob, Action setupAction, Action cleanupAction, long operationsPerInvoke, bool isDiagnoserAttached)
         {
             IdleAction = idleAction;
             MainAction = mainAction;
-            pilotStage = new EnginePilotStage(this);
+            TargetJob = targetJob;
+            SetupAction = setupAction;
+            CleanupAction = cleanupAction;
+            OperationsPerInvoke = operationsPerInvoke;
+            IsDiagnoserAttached = isDiagnoserAttached;
+
+            Resolver = new CompositeResolver(BenchmarkRunnerCore.DefaultResolver, EngineResolver.Instance);
+
+            Clock = targetJob.Infrastructure.Clock.Resolve(Resolver);
+            ForceAllocations = targetJob.Env.Gc.Force.Resolve(Resolver);
+            UnrollFactor = targetJob.Run.UnrollFactor.Resolve(Resolver);
+            Strategy = targetJob.Run.RunStrategy.Resolve(Resolver);
+            EvaluateOverhead = targetJob.Accuracy.EvaluateOverhead.Resolve(Resolver);
+            InvocationCount = targetJob.Run.InvocationCount.Resolve(Resolver);
+
             warmupStage = new EngineWarmupStage(this);
+            pilotStage = new EnginePilotStage(this);
             targetStage = new EngineTargetStage(this);
-            Resolver = new CompositeResolver(BenchmarkRunnerCore.DefaultResolver, EngineResolver.Instance);
         }
 
-        // TODO: return all measurements
-        [UsedImplicitly]
-        public void Run()
+        public IEngineFactory Factory => new EngineFactory();
+
+        public void PreAllocate()
         {
-            Jitting();
+            var list = new List<Measurement> { new Measurement(), new Measurement() };
+            list.Sort(); // provoke JIT, static ctors etc (was allocating 1740 bytes with first call)
+            if (TimeUnit.All == null || list[0].Nanoseconds != default(double))
+                throw new Exception("just use this things here to provoke static ctor");
+            isPreAllocated = true;
+        }
 
-            long invokeCount = 1;
-            int unrollFactor = TargetJob.Run.UnrollFactor.Resolve(Resolver);
-            IList<Measurement> idle = null;
+        public void Jitting()
+        {
+            // first signal about jitting is raised from auto-generated Program.cs, look at BenchmarkProgram.txt
+            MainAction.Invoke(1);
+            IdleAction.Invoke(1);
+            isJitted = true;
+        }
+
+        public RunResults Run()
+        {
+            if (!isJitted || !isPreAllocated)
+                throw new Exception("You must call PreAllocate() and Jitting() first!");
 
-            if (TargetJob.Run.RunStrategy.Resolve(Resolver) != RunStrategy.ColdStart)
+            long invokeCount = InvocationCount;
+            List<Measurement> idle = null;
+
+            if (Strategy != RunStrategy.ColdStart)
             {
                 invokeCount = pilotStage.Run();
 
-                if (TargetJob.Accuracy.EvaluateOverhead.Resolve(Resolver))
+                if (EvaluateOverhead)
                 {
-                    warmupStage.RunIdle(invokeCount, unrollFactor);
-                    idle = targetStage.RunIdle(invokeCount, unrollFactor);
+                    warmupStage.RunIdle(invokeCount, UnrollFactor);
+                    idle = targetStage.RunIdle(invokeCount, UnrollFactor);
                 }
 
-                warmupStage.RunMain(invokeCount, unrollFactor);
+                warmupStage.RunMain(invokeCount, UnrollFactor);
             }
-            var main = targetStage.RunMain(invokeCount, unrollFactor);
 
-            // TODO: Move calculation of the result measurements to a separated class
-            PrintResult(idle, main);
-        }
+            var main = targetStage.RunMain(invokeCount, UnrollFactor);
 
-        private void Jitting()
-        {
-            SetupAction?.Invoke();
-            MainAction.Invoke(1);
-            IdleAction.Invoke(1);
-            CleanupAction?.Invoke();
-        }
-
-        private void PrintResult(IList<Measurement> idle, IList<Measurement> main)
-        {
-            // TODO: use Accuracy.RemoveOutliers
-            // TODO: check if resulted measurements are too small (like < 0.1ns)
-            double overhead = idle == null ? 0.0 : new Statistics(idle.Select(m => m.Nanoseconds)).Median;
-            int resultIndex = 0;
-            foreach (var measurement in main)
-            {
-                var resultMeasurement = new Measurement(
-                    measurement.LaunchIndex,
-                    IterationMode.Result,
-                    ++resultIndex,
-                    measurement.Operations,
-                    Math.Max(0, measurement.Nanoseconds - overhead));
-                WriteLine(resultMeasurement.ToOutputLine());
-            }
-            WriteLine();
+            return new RunResults(idle, main);
         }
 
         public Measurement RunIteration(IterationData data)
@@ -103,38 +113,66 @@ public Measurement RunIteration(IterationData data)
             long totalOperations = invokeCount * OperationsPerInvoke;
             var action = data.IterationMode.IsIdle() ? IdleAction : MainAction;
 
-            // Setup
-            SetupAction?.Invoke();
             GcCollect();
 
             // Measure
-            var clock = TargetJob.Infrastructure.Clock.Resolve(Resolver).Start();
+            var clock = Clock.Start();
             action(invokeCount / unrollFactor);
             var clockSpan = clock.Stop();
 
-            // Cleanup
-            CleanupAction?.Invoke();
             GcCollect();
 
             // Results
             var measurement = new Measurement(0, data.IterationMode, data.Index, totalOperations, clockSpan.GetNanoseconds());
-            WriteLine(measurement.ToOutputLine());
+            if (!IsDiagnoserAttached) WriteLine(measurement.ToOutputLine());
+
             return measurement;
         }
 
-        private void GcCollect() => GcCollect(TargetJob.Env.Gc.Force.Resolve(Resolver));
-
-        private static void GcCollect(bool isForce)
+        private void GcCollect()
         {
-            if (!isForce)
+            if (!ForceAllocations)
                 return;
 
+            ForceGcCollect();
+        }
+
+        private static void ForceGcCollect()
+        {
             GC.Collect();
             GC.WaitForPendingFinalizers();
             GC.Collect();
         }
 
-        public void WriteLine() => Console.WriteLine();
-        public void WriteLine(string line) => Console.WriteLine(line);
+        public void WriteLine(string text)
+        {
+            EnsureNothingIsPrintedWhenDiagnoserIsAttached();
+
+            Console.WriteLine(text);
+        }
+
+        public void WriteLine()
+        {
+            EnsureNothingIsPrintedWhenDiagnoserIsAttached();
+
+            Console.WriteLine();
+        }
+
+        private void EnsureNothingIsPrintedWhenDiagnoserIsAttached()
+        {
+            if (IsDiagnoserAttached)
+            {
+                throw new InvalidOperationException("to avoid memory allocations we must not print anything when diagnoser is still attached");
+            }
+        }
+
+        [UsedImplicitly]
+        public class Signals
+        {
+            public const string BeforeAnythingElse = "// BeforeAnythingElse";
+            public const string AfterSetup = "// AfterSetup";
+            public const string BeforeCleanup = "// BeforeCleanup";
+            public const string DiagnoserIsAttachedParam = "diagnoserAttached";
+        }
     }
 }