Skip to content

Commit

Permalink
Engine.Run() should return the full list of performed measurements (f…
Browse files Browse the repository at this point in the history 
…ixes #2187) (#2188)

* Engine.Run() should return the full list of performed measurements (fixes #2187)

* Code review fixes
  • Loading branch information
@AndreyAkinshin
AndreyAkinshin committed Nov 3, 2022
1 parent 9e759f9 commit e75bdd8
Show file tree
Hide file tree
Showing 12 changed files with 186 additions and 80 deletions.
21 changes: 14 additions & 7 deletions src/BenchmarkDotNet/Engines/Engine.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ public class Engine : IEngine
private bool EvaluateOverhead { get; }
private bool MemoryRandomization { get; }

private readonly List<Measurement> jittingMeasurements = new (10);
private readonly EnginePilotStage pilotStage;
private readonly EngineWarmupStage warmupStage;
private readonly EngineActualStage actualStage;
Expand Down Expand Up @@ -104,8 +105,10 @@ public void Dispose()

public RunResults Run()
{
var measurements = new List<Measurement>();
measurements.AddRange(jittingMeasurements);

long invokeCount = TargetJob.ResolveValue(RunMode.InvocationCountCharacteristic, Resolver, 1);
IReadOnlyList<Measurement> idle = null;

if (EngineEventSource.Log.IsEnabled())
EngineEventSource.Log.BenchmarkStart(BenchmarkName);
Expand All @@ -114,21 +117,23 @@ public RunResults Run()
{
if (Strategy != RunStrategy.Monitoring)
{
invokeCount = pilotStage.Run();
var pilotStageResult = pilotStage.Run();
invokeCount = pilotStageResult.PerfectInvocationCount;
measurements.AddRange(pilotStageResult.Measurements);

if (EvaluateOverhead)
{
warmupStage.RunOverhead(invokeCount, UnrollFactor);
idle = actualStage.RunOverhead(invokeCount, UnrollFactor);
measurements.AddRange(warmupStage.RunOverhead(invokeCount, UnrollFactor));
measurements.AddRange(actualStage.RunOverhead(invokeCount, UnrollFactor));
}
}

warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy);
measurements.AddRange(warmupStage.RunWorkload(invokeCount, UnrollFactor, Strategy));
}

Host.BeforeMainRun();

var main = actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring);
measurements.AddRange(actualStage.RunWorkload(invokeCount, UnrollFactor, forceSpecific: Strategy == RunStrategy.Monitoring));

Host.AfterMainRun();

Expand All @@ -141,7 +146,7 @@ public RunResults Run()

var outlierMode = TargetJob.ResolveValue(AccuracyMode.OutlierModeCharacteristic, Resolver);

return new RunResults(idle, main, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
return new RunResults(measurements, outlierMode, workGcHasDone, threadingStats, exceptionFrequency);
}

public Measurement RunIteration(IterationData data)
Expand Down Expand Up @@ -183,6 +188,8 @@ public Measurement RunIteration(IterationData data)
// Results
var measurement = new Measurement(0, data.IterationMode, data.IterationStage, data.Index, totalOperations, clockSpan.GetNanoseconds());
WriteLine(measurement.ToString());
if (measurement.IterationStage == IterationStage.Jitting)
jittingMeasurements.Add(measurement);

Consume(stackMemory);

Expand Down
38 changes: 32 additions & 6 deletions src/BenchmarkDotNet/Engines/EnginePilotStage.cs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,35 @@
using System;
using System.Collections.Generic;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using JetBrains.Annotations;
using Perfolizer.Horology;

namespace BenchmarkDotNet.Engines
{
// TODO: use clockResolution
internal class EnginePilotStage : EngineStage
{
public readonly struct PilotStageResult
{
public long PerfectInvocationCount { get; }
[NotNull]
public IReadOnlyList<Measurement> Measurements { get; }

public PilotStageResult(long perfectInvocationCount, [NotNull] List<Measurement> measurements)
{
PerfectInvocationCount = perfectInvocationCount;
Measurements = measurements;
}

public PilotStageResult(long perfectInvocationCount)
{
PerfectInvocationCount = perfectInvocationCount;
Measurements = Array.Empty<Measurement>();
}
}

internal const long MaxInvokeCount = (long.MaxValue / 2 + 1) / 2;

private readonly int unrollFactor;
Expand All @@ -30,11 +52,11 @@ public EnginePilotStage(IEngine engine) : base(engine)
}

/// <returns>Perfect invocation count</returns>
public long Run()
public PilotStageResult Run()
{
// If InvocationCount is specified, pilot stage should be skipped
if (TargetJob.HasValue(RunMode.InvocationCountCharacteristic))
return TargetJob.Run.InvocationCount;
return new PilotStageResult(TargetJob.Run.InvocationCount);

// Here we want to guess "perfect" amount of invocation
return TargetJob.HasValue(RunMode.IterationTimeCharacteristic)
Expand All @@ -45,15 +67,17 @@ public long Run()
/// <summary>
/// A case where we don't have specific iteration time.
/// </summary>
private long RunAuto()
private PilotStageResult RunAuto()
{
long invokeCount = Autocorrect(minInvokeCount);
var measurements = new List<Measurement>();

int iterationCounter = 0;
while (true)
{
iterationCounter++;
var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
measurements.Add(measurement);
double iterationTime = measurement.Nanoseconds;
double operationError = 2.0 * resolution / invokeCount; // An operation error which has arisen due to the Chronometer precision

Expand All @@ -75,15 +99,16 @@ private long RunAuto()
}
WriteLine();

return invokeCount;
return new PilotStageResult(invokeCount, measurements);
}

/// <summary>
/// A case where we have specific iteration time.
/// </summary>
private long RunSpecific()
private PilotStageResult RunSpecific()
{
long invokeCount = Autocorrect(Engine.MinInvokeCount);
var measurements = new List<Measurement>();

int iterationCounter = 0;

Expand All @@ -92,6 +117,7 @@ private long RunSpecific()
{
iterationCounter++;
var measurement = RunIteration(IterationMode.Workload, IterationStage.Pilot, iterationCounter, invokeCount, unrollFactor);
measurements.Add(measurement);
double actualIterationTime = measurement.Nanoseconds;
long newInvokeCount = Autocorrect(Math.Max(minInvokeCount, (long)Math.Round(invokeCount * targetIterationTime / actualIterationTime)));

Expand All @@ -105,7 +131,7 @@ private long RunSpecific()
}
WriteLine();

return invokeCount;
return new PilotStageResult(invokeCount, measurements);
}

private long Autocorrect(long count) => (count + unrollFactor - 1) / unrollFactor * unrollFactor;
Expand Down
6 changes: 3 additions & 3 deletions src/BenchmarkDotNet/Engines/EngineWarmupStage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ internal class EngineWarmupStage : EngineStage

public EngineWarmupStage(IEngine engine) : base(engine) => this.engine = engine;

public void RunOverhead(long invokeCount, int unrollFactor)
public IReadOnlyList<Measurement> RunOverhead(long invokeCount, int unrollFactor)
=> Run(invokeCount, IterationMode.Overhead, unrollFactor, RunStrategy.Throughput);

public void RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
public IReadOnlyList<Measurement> RunWorkload(long invokeCount, int unrollFactor, RunStrategy runStrategy)
=> Run(invokeCount, IterationMode.Workload, unrollFactor, runStrategy);

internal List<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
internal IReadOnlyList<Measurement> Run(long invokeCount, IterationMode iterationMode, int unrollFactor, RunStrategy runStrategy)
{
var criteria = DefaultStoppingCriteriaFactory.Instance.CreateWarmup(engine.TargetJob, engine.Resolver, iterationMode, runStrategy);
return Run(criteria, invokeCount, iterationMode, IterationStage.Warmup, unrollFactor);
Expand Down
51 changes: 36 additions & 15 deletions src/BenchmarkDotNet/Engines/RunResults.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Mathematics;
using BenchmarkDotNet.Reports;
using JetBrains.Annotations;
Expand All @@ -13,39 +14,51 @@ public struct RunResults
{
private readonly OutlierMode outlierMode;

[NotNull, PublicAPI]
public IReadOnlyList<Measurement> EngineMeasurements { get; }

[CanBeNull, PublicAPI]
public IReadOnlyList<Measurement> Overhead { get; }
public IReadOnlyList<Measurement> Overhead
=> EngineMeasurements
.Where(m => m.Is(IterationMode.Overhead, IterationStage.Actual))
.ToArray();

[NotNull, PublicAPI]
public IReadOnlyList<Measurement> Workload { get; }
public IReadOnlyList<Measurement> Workload
=> EngineMeasurements
.Where(m => m.Is(IterationMode.Workload, IterationStage.Actual))
.ToArray();

public GcStats GCStats { get; }

public ThreadingStats ThreadingStats { get; }

public double ExceptionFrequency { get; }

public RunResults([CanBeNull] IReadOnlyList<Measurement> overhead,
[NotNull] IReadOnlyList<Measurement> workload,
OutlierMode outlierMode,
GcStats gcStats,
ThreadingStats threadingStats,
double exceptionFrequency)
public RunResults([NotNull] IReadOnlyList<Measurement> engineMeasurements,
OutlierMode outlierMode,
GcStats gcStats,
ThreadingStats threadingStats,
double exceptionFrequency)
{
this.outlierMode = outlierMode;
Overhead = overhead;
Workload = workload;
EngineMeasurements = engineMeasurements;
GCStats = gcStats;
ThreadingStats = threadingStats;
ExceptionFrequency = exceptionFrequency;
}

public IEnumerable<Measurement> GetMeasurements()
public IEnumerable<Measurement> GetWorkloadResultMeasurements()
{
double overhead = Overhead == null ? 0.0 : new Statistics(Overhead.Select(m => m.Nanoseconds)).Median;
var mainStats = new Statistics(Workload.Select(m => m.Nanoseconds));
var overheadActualMeasurements = Overhead ?? Array.Empty<Measurement>();
var workloadActualMeasurements = Workload;
if (workloadActualMeasurements.IsEmpty())
yield break;

double overhead = overheadActualMeasurements.IsEmpty() ? 0.0 : new Statistics(overheadActualMeasurements.Select(m => m.Nanoseconds)).Median;
var mainStats = new Statistics(workloadActualMeasurements.Select(m => m.Nanoseconds));
int resultIndex = 0;
foreach (var measurement in Workload)
foreach (var measurement in workloadActualMeasurements)
{
if (mainStats.IsActualOutlier(measurement.Nanoseconds, outlierMode))
continue;
Expand All @@ -63,9 +76,17 @@ public IEnumerable<Measurement> GetMeasurements()
}
}

public IEnumerable<Measurement> GetAllMeasurements()
{
foreach (var measurement in EngineMeasurements)
yield return measurement;
foreach (var measurement in GetWorkloadResultMeasurements())
yield return measurement;
}

public void Print(TextWriter outWriter)
{
foreach (var measurement in GetMeasurements())
foreach (var measurement in GetWorkloadResultMeasurements())
outWriter.WriteLine(measurement.ToString());

if (!GCStats.Equals(GcStats.Empty))
Expand Down
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet/Toolchains/Results/ExecuteResult.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ internal ExecuteResult(List<Measurement> measurements, GcStats gcStats, Threadin
internal static ExecuteResult FromRunResults(RunResults runResults, int exitCode)
=> exitCode != 0
? CreateFailed(exitCode)
: new ExecuteResult(runResults.GetMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);
: new ExecuteResult(runResults.GetAllMeasurements().ToList(), runResults.GCStats, runResults.ThreadingStats, runResults.ExceptionFrequency);

internal static ExecuteResult CreateFailed(int exitCode = -1)
=> new ExecuteResult(false, exitCode, default, Array.Empty<string>(), Array.Empty<string>(), Array.Empty<string>(), 0);
Expand Down
7 changes: 5 additions & 2 deletions tests/BenchmarkDotNet.IntegrationTests/CustomEngineTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,11 @@ public RunResults Run()
Console.WriteLine(EngineRunMessage);

return new RunResults(
new List<Measurement> { new Measurement(1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1) },
new List<Measurement> { new Measurement(1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1) },
new List<Measurement>
{
new (1, IterationMode.Overhead, IterationStage.Actual, 1, 1, 1),
new (1, IterationMode.Workload, IterationStage.Actual, 1, 1, 1)
},
OutlierMode.DontRemove,
default,
default,
Expand Down
78 changes: 78 additions & 0 deletions tests/BenchmarkDotNet.IntegrationTests/EngineTests.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
using System.Linq;
using System.Threading;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Engines;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Reports;
using Xunit;
using Xunit.Abstractions;

namespace BenchmarkDotNet.IntegrationTests
{
public class EngineTests : BenchmarkTestExecutor
{
public EngineTests(ITestOutputHelper output) : base(output) { }

[Fact]
public void ZeroWarmupCountIsApplied()
{
var job = Job.InProcess
.WithEvaluateOverhead(false)
.WithWarmupCount(0)
.WithIterationCount(1)
.WithInvocationCount(1)
.WithUnrollFactor(1);
var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
var summary = CanExecute<FooBench>(config);
var report = summary.Reports.Single();
int workloadWarmupCount = report.AllMeasurements
.Count(m => m.Is(IterationMode.Workload, IterationStage.Warmup));
Assert.Equal(0, workloadWarmupCount);
}

[Fact]
public void AllMeasurementsArePerformedDefault() => AllMeasurementsArePerformed(Job.Default);

[Fact]
public void AllMeasurementsArePerformedInProcess() => AllMeasurementsArePerformed(Job.InProcess);

private void AllMeasurementsArePerformed(Job baseJob)
{
var job = baseJob
.WithWarmupCount(1)
.WithIterationCount(1)
.WithInvocationCount(1)
.WithUnrollFactor(1);
var config = DefaultConfig.Instance.AddJob(job).WithOptions(ConfigOptions.DisableOptimizationsValidator);
var summary = CanExecute<FooBench>(config);
var measurements = summary.Reports.Single().AllMeasurements;

Output.WriteLine("*** AllMeasurements ***");
foreach (var measurement in measurements)
Output.WriteLine(measurement.ToString());
Output.WriteLine("-----");

void Check(IterationMode mode, IterationStage stage)
{
int count = measurements.Count(m => m.Is(mode, stage));
Output.WriteLine($"Count({mode}{stage}) = {count}");
Assert.True(count > 0, $"AllMeasurements don't contain {mode}{stage}");
}

Check(IterationMode.Overhead, IterationStage.Jitting);
Check(IterationMode.Workload, IterationStage.Jitting);
Check(IterationMode.Overhead, IterationStage.Warmup);
Check(IterationMode.Overhead, IterationStage.Actual);
Check(IterationMode.Workload, IterationStage.Warmup);
Check(IterationMode.Workload, IterationStage.Actual);
Check(IterationMode.Workload, IterationStage.Result);
}

public class FooBench
{
[Benchmark]
public void Foo() => Thread.Sleep(10);
}
}
}
Loading

0 comments on commit e75bdd8

Please sign in to comment.