Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit e919c04

Browse files
AndyAyersMSAndyAyersMS
committed
Update jitbench harness to capture steady state data
Update jitbench SHA to pick up new version that reports steady state median response time, with fractional digits.
1 parent afb65d4 commit e919c04

File tree

2 files changed

+86
-27
lines changed

2 files changed

+86
-27
lines changed

‎tests/src/performance/Scenario/JitBench/IterationData.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,7 @@ internal class IterationData
2020
public double FirstRequestTime { get; set; }
2121

2222
public double SteadystateTime { get; set; }
23+
24+
public double SteadystateMedianTime { get; set; }
2325
}
2426
}

‎tests/src/performance/Scenario/JitBench/JitBenchHarness.cs‎

Lines changed: 84 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@ static void Main(string[] args)
3939
startInfo.Environment.Add("COMPlus_EXPERIMENTAL_TieredCompilation", "1");
4040
scenarioName += " Tiering";
4141
}
42-
4342
if (options.Minopts)
4443
{
4544
startInfo.Environment.Add("COMPlus_JITMinOpts", "1");
@@ -58,10 +57,13 @@ static void Main(string[] args)
5857
scenarioName += " NoNgen";
5958
}
6059

60+
PrintHeader($"Running scenario '{scenarioName}'");
61+
6162
var program = new JitBenchHarness();
6263
try
6364
{
64-
var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(60000), startInfo) {
65+
var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMilliseconds(60000), startInfo)
66+
{
6567
Iterations = (int)options.Iterations,
6668
PreIterationDelegate = program.PreIteration,
6769
PostIterationDelegate = program.PostIteration,
@@ -86,7 +88,8 @@ static void Main(string[] args)
8688
if (!Directory.Exists(startInfo.WorkingDirectory))
8789
throw new DirectoryNotFoundException(startInfo.WorkingDirectory);
8890

89-
h.RunScenario(scenarioConfiguration, teardownDelegate: (ScenarioBenchmark scenarioBenchmark) => {
91+
h.RunScenario(scenarioConfiguration, teardownDelegate: (ScenarioBenchmark scenarioBenchmark) =>
92+
{
9093
program.PostRun(scenarioBenchmark, "MusicStore", processesOfInterest, modulesOfInterest);
9194
});
9295
}
@@ -157,7 +160,8 @@ private static void DownloadAndExtractJitBenchRepo()
157160

158161
private static void InstallSharedRuntime()
159162
{
160-
var psi = new ProcessStartInfo {
163+
var psi = new ProcessStartInfo
164+
{
161165
WorkingDirectory = s_jitBenchDevDirectory,
162166
FileName = @"powershell.exe",
163167
Arguments = $".\\Dotnet-Install.ps1 -SharedRuntime -InstallDir .dotnet -Channel master -Architecture {s_targetArchitecture}"
@@ -167,7 +171,8 @@ private static void InstallSharedRuntime()
167171

168172
private static IDictionary<string, string> InstallDotnet()
169173
{
170-
var psi = new ProcessStartInfo {
174+
var psi = new ProcessStartInfo
175+
{
171176
WorkingDirectory = s_jitBenchDevDirectory,
172177
FileName = @"powershell.exe",
173178
Arguments = $".\\Dotnet-Install.ps1 -InstallDir .dotnet -Channel master -Architecture {s_targetArchitecture}"
@@ -225,7 +230,8 @@ private static void ModifySharedFramework()
225230
private static IDictionary<string, string> GenerateStore(IDictionary<string, string> environment)
226231
{
227232
// This step generates some environment variables needed later.
228-
var psi = new ProcessStartInfo {
233+
var psi = new ProcessStartInfo
234+
{
229235
WorkingDirectory = s_jitBenchDevDirectory,
230236
FileName = "powershell.exe",
231237
Arguments = $"-Command \".\\AspNet-GenerateStore.ps1 -InstallDir .store -Architecture {s_targetArchitecture} -Runtime win7-{s_targetArchitecture}; gi env:JITBENCH_*, env:DOTNET_SHARED_STORE | %{{ \\\"$($_.Name)=$($_.Value)\\\" }} 1>>{EnvironmentFileName}\""
@@ -259,7 +265,8 @@ private static IDictionary<string, string> GetEnvironment(IDictionary<string, st
259265

260266
private static void DotNetInfo(string workingDirectory, string dotnetFileName, IDictionary<string, string> environment)
261267
{
262-
var psi = new ProcessStartInfo {
268+
var psi = new ProcessStartInfo
269+
{
263270
WorkingDirectory = workingDirectory,
264271
FileName = dotnetFileName,
265272
Arguments = "--info"
@@ -270,7 +277,8 @@ private static void DotNetInfo(string workingDirectory, string dotnetFileName, I
270277

271278
private static void RestoreMusicStore(string workingDirectory, string dotnetFileName, IDictionary<string, string> environment)
272279
{
273-
var psi = new ProcessStartInfo {
280+
var psi = new ProcessStartInfo
281+
{
274282
WorkingDirectory = workingDirectory,
275283
FileName = dotnetFileName,
276284
Arguments = "restore"
@@ -285,7 +293,8 @@ private static void PublishMusicStore(string workingDirectory, string dotnetFile
285293
if (!File.Exists(manifest))
286294
throw new FileNotFoundException(manifest);
287295

288-
var psi = new ProcessStartInfo {
296+
var psi = new ProcessStartInfo
297+
{
289298
WorkingDirectory = workingDirectory,
290299
FileName = dotnetFileName,
291300
Arguments = $"publish -c Release -f {JitBenchTargetFramework} --manifest \"{manifest}\" /p:MvcRazorCompileOnPublish=false -o \"{MusicStorePublishDirectory}\""
@@ -316,7 +325,8 @@ private static IDictionary<string, string> GetInitialEnvironment()
316325

317326
private static ProcessStartInfo CreateJitBenchStartInfo(IDictionary<string, string> environment)
318327
{
319-
var psi = new ProcessStartInfo {
328+
var psi = new ProcessStartInfo
329+
{
320330
Arguments = "MusicStore.dll",
321331
FileName = s_dotnetProcessFileName,
322332
RedirectStandardError = true,
@@ -385,7 +395,7 @@ private static void ValidateEnvironment(IDictionary<string, string> environment)
385395
}
386396

387397
private const string JitBenchRepoUrl = "https://github.com/aspnet/JitBench";
388-
private const string JitBenchCommitSha1Id = "b7e7b786c60daa255aacaea85006afe4d4ec8306";
398+
private const string JitBenchCommitSha1Id = "36db11e7ab15e96af10d995a048a1476b4e73d43";
389399
private const string JitBenchTargetFramework = "netcoreapp2.1";
390400
private const string EnvironmentFileName = "JitBenchEnvironment.txt";
391401

@@ -398,7 +408,8 @@ private void PreIteration(ScenarioTest scenario)
398408

399409
if (scenario.Process.StartInfo.RedirectStandardError)
400410
{
401-
scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) => {
411+
scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) =>
412+
{
402413
if (!string.IsNullOrEmpty(errorLine.Data))
403414
_stderr.AppendLine(errorLine.Data);
404415
};
@@ -409,7 +420,8 @@ private void PreIteration(ScenarioTest scenario)
409420

410421
if (scenario.Process.StartInfo.RedirectStandardOutput)
411422
{
412-
scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) => {
423+
scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) =>
424+
{
413425
if (!string.IsNullOrEmpty(outputLine.Data))
414426
_stdout.AppendLine(outputLine.Data);
415427
};
@@ -423,6 +435,7 @@ private void PostIteration(ScenarioExecutionResult scenarioExecutionResult)
423435
double? startupTime = null;
424436
double? firstRequestTime = null;
425437
double? steadyStateAverageTime = null;
438+
double? steadyStateMedianTime = null;
426439

427440
using (var reader = new StringReader(_stdout.ToString()))
428441
{
@@ -443,11 +456,18 @@ private void PostIteration(ScenarioExecutionResult scenarioExecutionResult)
443456
continue;
444457
}
445458

446-
match = Regex.Match(line, @"^Steadystate average response time: (\d+)ms$");
459+
match = Regex.Match(line, @"^Steadystate average response time: (\d+\.?\d*)ms$");
447460
if (match.Success && match.Groups.Count == 2)
448461
{
449462
steadyStateAverageTime = Convert.ToDouble(match.Groups[1].Value);
450-
break;
463+
continue;
464+
}
465+
466+
match = Regex.Match(line, @"^Steadystate median response time: (\d+\.?\d*)ms$");
467+
if (match.Success && match.Groups.Count == 2)
468+
{
469+
steadyStateMedianTime = Convert.ToDouble(match.Groups[1].Value);
470+
continue;
451471
}
452472
}
453473
}
@@ -458,19 +478,24 @@ private void PostIteration(ScenarioExecutionResult scenarioExecutionResult)
458478
throw new Exception("First Request time was not found.");
459479
if (!steadyStateAverageTime.HasValue)
460480
throw new Exception("Steady state average response time not found.");
481+
if (!steadyStateMedianTime.HasValue)
482+
throw new Exception("Steady state median response time not found.");
461483

462-
IterationsData.Add(new IterationData {
484+
IterationsData.Add(new IterationData
485+
{
463486
ScenarioExecutionResult = scenarioExecutionResult,
464487
StandardOutput = _stdout.ToString(),
465488
StartupTime = startupTime.Value,
466489
FirstRequestTime = firstRequestTime.Value,
467490
SteadystateTime = steadyStateAverageTime.Value,
491+
SteadystateMedianTime = steadyStateMedianTime.Value,
468492
});
469493

470494
PrintRunningStepInformation($"({IterationsData.Count}) Server started in {IterationsData.Last().StartupTime}ms");
471495
PrintRunningStepInformation($"({IterationsData.Count}) Request took {IterationsData.Last().FirstRequestTime}ms");
472496
PrintRunningStepInformation($"({IterationsData.Count}) Cold start time (server start + first request time): {IterationsData.Last().StartupTime + IterationsData.Last().FirstRequestTime}ms");
473497
PrintRunningStepInformation($"({IterationsData.Count}) Average steady state response {IterationsData.Last().SteadystateTime}ms");
498+
PrintRunningStepInformation($"({IterationsData.Count}) Median steady state response {IterationsData.Last().SteadystateMedianTime}ms");
474499

475500
_stdout.Clear();
476501
_stderr.Clear();
@@ -499,7 +524,8 @@ private void PostRun(
499524
scenarioTestModel.Performance.Metrics.Add(ElapsedTimeMilliseconds);
500525
}
501526

502-
scenarioTestModel.Performance.IterationModels.Add(new IterationModel {
527+
scenarioTestModel.Performance.IterationModels.Add(new IterationModel
528+
{
503529
Iteration = new Dictionary<string, double> {
504530
{ ElapsedTimeMilliseconds.Name, (scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime).TotalMilliseconds},
505531
}
@@ -510,7 +536,8 @@ private void PostRun(
510536
.SingleOrDefault(t => t.Name == "Startup" && t.Namespace == scenarioTestModel.Name);
511537
if (startup == null)
512538
{
513-
startup = new ScenarioTestModel("Startup") {
539+
startup = new ScenarioTestModel("Startup")
540+
{
514541
Namespace = scenarioTestModel.Name,
515542
};
516543
scenarioBenchmark.Tests.Add(startup);
@@ -523,7 +550,8 @@ private void PostRun(
523550
.SingleOrDefault(t => t.Name == "First Request" && t.Namespace == scenarioTestModel.Name);
524551
if (firstRequest == null)
525552
{
526-
firstRequest = new ScenarioTestModel("First Request") {
553+
firstRequest = new ScenarioTestModel("First Request")
554+
{
527555
Namespace = scenarioTestModel.Name,
528556
};
529557
scenarioBenchmark.Tests.Add(firstRequest);
@@ -532,18 +560,41 @@ private void PostRun(
532560
firstRequest.Performance.Metrics.Add(ElapsedTimeMilliseconds);
533561
}
534562

535-
startup.Performance.IterationModels.Add(new IterationModel {
563+
var medianResponse = scenarioBenchmark.Tests
564+
.SingleOrDefault(t => t.Name == "Median Response" && t.Namespace == scenarioTestModel.Name);
565+
if (medianResponse == null)
566+
{
567+
medianResponse = new ScenarioTestModel("Median Response")
568+
{
569+
Namespace = scenarioTestModel.Name,
570+
};
571+
scenarioBenchmark.Tests.Add(medianResponse);
572+
573+
// Add measured metrics to each test.
574+
medianResponse.Performance.Metrics.Add(ElapsedTimeMilliseconds);
575+
}
576+
577+
startup.Performance.IterationModels.Add(new IterationModel
578+
{
536579
Iteration = new Dictionary<string, double> {
537580
{ ElapsedTimeMilliseconds.Name, iter.StartupTime },
538581
},
539582
});
540583

541-
firstRequest.Performance.IterationModels.Add(new IterationModel {
584+
firstRequest.Performance.IterationModels.Add(new IterationModel
585+
{
542586
Iteration = new Dictionary<string, double> {
543587
{ ElapsedTimeMilliseconds.Name, iter.FirstRequestTime },
544588
},
545589
});
546590

591+
medianResponse.Performance.IterationModels.Add(new IterationModel
592+
{
593+
Iteration = new Dictionary<string, double> {
594+
{ ElapsedTimeMilliseconds.Name, iter.SteadystateMedianTime },
595+
},
596+
});
597+
547598
if (!string.IsNullOrWhiteSpace(iter.ScenarioExecutionResult.EventLogFileName) &&
548599
File.Exists(iter.ScenarioExecutionResult.EventLogFileName))
549600
{
@@ -561,7 +612,8 @@ private static ScenarioBenchmark AddEtwData(
561612
IReadOnlyCollection<string> modulesOfInterest)
562613
{
563614
var metricModels = scenarioExecutionResult.PerformanceMonitorCounters
564-
.Select(pmc => new MetricModel {
615+
.Select(pmc => new MetricModel
616+
{
565617
DisplayName = pmc.DisplayName,
566618
Name = pmc.Name,
567619
Unit = pmc.Unit,
@@ -581,7 +633,8 @@ private static ScenarioBenchmark AddEtwData(
581633
.SingleOrDefault(t => t.Name == process.Name && t.Namespace == "");
582634
if (processTest == null)
583635
{
584-
processTest = new ScenarioTestModel(process.Name) {
636+
processTest = new ScenarioTestModel(process.Name)
637+
{
585638
Namespace = "",
586639
};
587640
scenarioBenchmark.Tests.Add(processTest);
@@ -591,7 +644,8 @@ private static ScenarioBenchmark AddEtwData(
591644
processTest.Performance.Metrics.AddRange(metricModels);
592645
}
593646

594-
var processIterationModel = new IterationModel {
647+
var processIterationModel = new IterationModel
648+
{
595649
Iteration = new Dictionary<string, double>()
596650
};
597651
processTest.Performance.IterationModels.Add(processIterationModel);
@@ -614,7 +668,8 @@ private static ScenarioBenchmark AddEtwData(
614668

615669
if (moduleTest == null)
616670
{
617-
moduleTest = new ScenarioTestModel(moduleTestName) {
671+
moduleTest = new ScenarioTestModel(moduleTestName)
672+
{
618673
Namespace = process.Name,
619674
Separator = "!",
620675
};
@@ -624,7 +679,8 @@ private static ScenarioBenchmark AddEtwData(
624679
moduleTest.Performance.Metrics.AddRange(metricModels);
625680
}
626681

627-
var moduleIterationModel = new IterationModel {
682+
var moduleIterationModel = new IterationModel
683+
{
628684
Iteration = new Dictionary<string, double>()
629685
};
630686
moduleTest.Performance.IterationModels.Add(moduleIterationModel);
@@ -686,7 +742,8 @@ private static void PrintRunningStepInformation(string message)
686742

687743
private List<IterationData> IterationsData { get; }
688744

689-
private static MetricModel ElapsedTimeMilliseconds { get; } = new MetricModel {
745+
private static MetricModel ElapsedTimeMilliseconds { get; } = new MetricModel
746+
{
690747
DisplayName = "Duration",
691748
Name = "Duration",
692749
Unit = "ms",

0 commit comments

Comments
 (0)