Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit c5e25ee

Browse files
michellemcdanielmichellemcdaniel
committed
Add JitMinOpts throughput jobs
This change adds an opt_level throughput option so that we can run throughput jobs in minopts mode. This also adds jobs to the lab for min opts runs
1 parent 214d82d commit c5e25ee

File tree

2 files changed

+149
-113
lines changed

2 files changed

+149
-113
lines changed

‎perf.groovy‎

Lines changed: 127 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -167,68 +167,75 @@ def static getOSGroup(def os) {
167167
[true, false].each { isPR ->
168168
['Windows_NT'].each { os ->
169169
['x64', 'x86'].each { arch ->
170-
def architecture = arch
171-
172-
def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}", isPR)) {
173-
// Set the label.
174-
label('windows_clr_perf')
175-
wrappers {
176-
credentialsBinding {
177-
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
170+
['full_opt', 'min_opt'].each { opt_level ->
171+
def architecture = arch
172+
173+
def newJob = job(Utilities.getFullJobName(project, "perf_throughput_perflab_${os}_${arch}_${opt_level}", isPR)) {
174+
// Set the label.
175+
label('windows_clr_perf')
176+
wrappers {
177+
credentialsBinding {
178+
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
179+
}
178180
}
179-
}
180181

181-
if (isPR)
182-
{
183-
parameters
184-
{
185-
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
186-
}
187-
}
188-
def configuration = 'Release'
189-
def runType = isPR ? 'private' : 'rolling'
190-
def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
182+
if (isPR)
183+
{
184+
parameters
185+
{
186+
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
187+
}
188+
}
189+
def configuration = 'Release'
190+
def runType = isPR ? 'private' : 'rolling'
191+
def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
191192

192-
steps {
193-
// Batch
194-
195-
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
196-
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
197-
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
198-
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
199-
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
200-
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
201-
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
202-
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
203-
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
204-
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
205-
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
206-
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
207-
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
208-
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
209-
batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
193+
steps {
194+
// Batch
195+
196+
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\"")
197+
batchFile("if exist \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\" rmdir /s /q \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\"")
198+
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
199+
batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
200+
//Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
201+
//we have to do it all as one statement because cmd is called each time and we lose the set environment variable
202+
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
203+
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
204+
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
205+
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
206+
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
207+
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
208+
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture} skiptests")
209+
batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
210+
batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
211+
}
210212
}
211-
}
212213

213-
// Save machinedata.json to /artifact/bin/ Jenkins dir
214-
def archiveSettings = new ArchivalSettings()
215-
archiveSettings.addFiles('throughput-*.csv')
216-
Utilities.addArchival(newJob, archiveSettings)
214+
// Save machinedata.json to /artifact/bin/ Jenkins dir
215+
def archiveSettings = new ArchivalSettings()
216+
archiveSettings.addFiles('throughput-*.csv')
217+
Utilities.addArchival(newJob, archiveSettings)
217218

218-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
219+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
219220

220-
if (isPR) {
221-
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
222-
builder.setGithubContext("${os} ${arch} CoreCLR Throughput Perf Tests")
223-
builder.triggerOnlyOnComment()
224-
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}\\W+throughput.*")
225-
builder.triggerForBranch(branch)
226-
builder.emitTrigger(newJob)
227-
}
228-
else {
229-
// Set a push trigger
230-
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
231-
builder.emitTrigger(newJob)
221+
if (isPR) {
222+
def opts = ""
223+
if (opt_level == 'min_opts')
224+
{
225+
opts = '\\W+min_opts'
226+
}
227+
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
228+
builder.setGithubContext("${os} ${arch} ${opt_level} CoreCLR Throughput Perf Tests")
229+
builder.triggerOnlyOnComment()
230+
builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+${arch}${opts}\\W+throughput.*")
231+
builder.triggerForBranch(branch)
232+
builder.emitTrigger(newJob)
233+
}
234+
else {
235+
// Set a push trigger
236+
TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
237+
builder.emitTrigger(newJob)
238+
}
232239
}
233240
}
234241
}
@@ -397,76 +404,89 @@ def static getFullThroughputJobName(def project, def os, def isPR) {
397404

398405
// Actual perf testing on the following OSes
399406
def throughputOSList = ['Ubuntu14.04']
407+
def throughputOptLevelList = ['full_opt', 'min_opt']
408+
409+
def throughputOSOptLevelList = []
410+
400411
throughputOSList.each { os ->
401-
def newJob = job(getFullThroughputJobName(project, os, isPR)) {
412+
throughputOptLevelList.each { opt_level ->
413+
throughputOSOptLevelList.add("${os}_${opt_level}")
414+
}
415+
}
402416

403-
label('linux_clr_perf')
404-
wrappers {
405-
credentialsBinding {
406-
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
417+
throughputOSList.each { os ->
418+
throughputOptLevelList.each { opt_level ->
419+
def newJob = job(getFullThroughputJobName(project, "${os}_${opt_level}", isPR)) {
420+
421+
label('linux_clr_perf')
422+
wrappers {
423+
credentialsBinding {
424+
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
425+
}
407426
}
408-
}
409427

410-
if (isPR)
411-
{
412-
parameters
428+
if (isPR)
413429
{
414-
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
430+
parameters
431+
{
432+
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that will be used to build the full title of a run in Benchview.')
433+
}
415434
}
416-
}
417435

418-
parameters {
419-
stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
420-
}
436+
parameters {
437+
stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
438+
}
421439

422-
def osGroup = getOSGroup(os)
423-
def runType = isPR ? 'private' : 'rolling'
424-
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
440+
def osGroup = getOSGroup(os)
441+
def runType = isPR ? 'private' : 'rolling'
442+
def benchViewName = isPR ? 'coreclr-throughput private \$BenchviewCommitName' : 'coreclr-throughput rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
425443

426-
steps {
427-
shell("bash ./tests/scripts/perf-prep.sh --throughput")
428-
shell("./init-tools.sh")
429-
copyArtifacts(fullBuildJobName) {
430-
includePatterns("bin/Product/**")
431-
buildSelector {
432-
buildNumber('\${PRODUCT_BUILD}')
444+
steps {
445+
shell("bash ./tests/scripts/perf-prep.sh --throughput")
446+
shell("./init-tools.sh")
447+
copyArtifacts(fullBuildJobName) {
448+
includePatterns("bin/Product/**")
449+
buildSelector {
450+
buildNumber('\${PRODUCT_BUILD}')
451+
}
433452
}
453+
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
454+
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
455+
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
456+
shell("""python3.5 ./tests/scripts/run-throughput-perf.py \\
457+
-arch \"${architecture}\" \\
458+
-os \"${os}\" \\
459+
-configuration \"${configuration}\" \\
460+
-opt_level \"${opt_level}\" \\
461+
-clr_root \"\${WORKSPACE}\" \\
462+
-assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
463+
-run_type \"${runType}\" \\
464+
-benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
434465
}
435-
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
436-
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
437-
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
438-
shell("""sudo -E python3.5 ./tests/scripts/run-throughput-perf.py \\
439-
-arch \"${architecture}\" \\
440-
-os \"${os}\" \\
441-
-configuration \"${configuration}\" \\
442-
-clr_root \"\${WORKSPACE}\" \\
443-
-assembly_root \"\${WORKSPACE}/Microsoft.Benchview.ThroughputBenchmarks.${architecture}.Windows_NT/lib\" \\
444-
-run_type \"${runType}\" \\
445-
-benchview_path \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\"""")
446466
}
447-
}
448467

449-
// Save machinedata.json to /artifact/bin/ Jenkins dir
450-
def archiveSettings = new ArchivalSettings()
451-
archiveSettings.addFiles('throughput-*.csv')
452-
archiveSettings.addFiles('machinedata.json')
453-
Utilities.addArchival(newJob, archiveSettings)
468+
// Save machinedata.json to /artifact/bin/ Jenkins dir
469+
def archiveSettings = new ArchivalSettings()
470+
archiveSettings.addFiles('throughput-*.csv')
471+
archiveSettings.addFiles('machinedata.json')
472+
Utilities.addArchival(newJob, archiveSettings)
454473

455-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
474+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
456475

457-
// For perf, we need to keep the run results longer
458-
newJob.with {
459-
// Enable the log rotator
460-
logRotator {
461-
artifactDaysToKeep(7)
462-
daysToKeep(300)
463-
artifactNumToKeep(25)
464-
numToKeep(1000)
476+
// For perf, we need to keep the run results longer
477+
newJob.with {
478+
// Enable the log rotator
479+
logRotator {
480+
artifactDaysToKeep(7)
481+
daysToKeep(300)
482+
artifactNumToKeep(25)
483+
numToKeep(1000)
484+
}
465485
}
466-
}
486+
} // opt_level
467487
} // os
468488

469-
def flowJobTPRunList = throughputOSList.collect { os ->
489+
def flowJobTPRunList = throughputOSOptLevelList.collect { os ->
470490
"{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
471491
}
472492
def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {

0 commit comments

Comments
 (0)