Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Enable linux perf runs #11905

Merged
merged 22 commits into from
May 27, 2017
Merged

Enable linux perf runs #11905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
0a6bbde
Removing `[assembly: MeasureInstructionsRetired]` from performance te…
May 23, 2017
948eb65
Updating bash script to run Linux perf tests.
May 23, 2017
f78722e
Updating run-xunit-perf.sh
May 24, 2017
402bb4c
Fixing overlay copy.
May 24, 2017
d1c7e7f
Updating the version of xUnit Performance API that has the Linux bug …
May 24, 2017
b6a6b14
Small modifications to shell script
May 24, 2017
f673bf2
Small modifications to shell script
May 24, 2017
8adf3a7
Enable running benchview tools if path specified.
May 24, 2017
7904056
Fixing typo.
May 24, 2017
262a926
Fixing python command.
May 24, 2017
4bbde51
More bash script updates for Linux.
May 24, 2017
1290fae
Minor changes to the bash script.
May 24, 2017
dd59ee1
Bug fix and minor changes.
May 24, 2017
316d63c
Bug fix
May 24, 2017
05e3f27
Adding the option to execute CoreRun with the stability prefix.
May 24, 2017
e4b033e
Minor changes.
May 24, 2017
1c5578d
Adding quotes around the run_command execution wrapper
May 25, 2017
b493735
Moving file to be archived by Jenkins and updating groovy file.
May 25, 2017
0b8cc10
Fix typo.
May 25, 2017
65b2040
Fixing the file names (artifacts) that need to be uploaded.
May 25, 2017
1ab6a7a
Escaping $ sign.
May 25, 2017
51c6353
Setting the maximum number of iterations.
May 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 50 additions & 39 deletions perf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ def static getOSGroup(def os) {
assert osGroup != null : "Could not find os group for ${os}"
return osGroupMap[os]
}

// Setup perflab tests runs
[true, false].each { isPR ->
['Windows_NT'].each { os ->
Expand Down Expand Up @@ -51,34 +52,34 @@ def static getOSGroup(def os) {
}
}

if (isPR)
{
parameters
if (isPR)
{
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
parameters
{
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
}
}
}
if (isSmoketest)
{
parameters
if (isSmoketest)
{
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
parameters
{
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing')
}
}
}
else
{
parameters
else
{
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
parameters
{
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
}
}
}
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
def uploadString = isSmoketest ? '' : '-uploadToBenchview'

def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'
def uploadString = isSmoketest ? '' : '-uploadToBenchview'

steps {
// Batch

Expand Down Expand Up @@ -114,7 +115,7 @@ def static getOSGroup(def os) {
batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"")
}
}

if (isSmoketest)
{
Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated')
Expand All @@ -123,18 +124,19 @@ def static getOSGroup(def os) {
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('Perf-*.xml')
archiveSettings.addFiles('Perf-*.etl')
archiveSettings.addFiles('machinedata.json')
Utilities.addArchival(newJob, archiveSettings)

Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")

newJob.with {
wrappers {
timeout {
absolute(240)
}
}
}

if (isPR) {
TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
if (isSmoketest)
Expand Down Expand Up @@ -185,7 +187,7 @@ def static getOSGroup(def os) {
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%'

steps {
// Batch

Expand Down Expand Up @@ -235,49 +237,58 @@ def static getOSGroup(def os) {
[true, false].each { isPR ->
['Ubuntu14.04'].each { os ->
def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {

label('linux_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}

}

if (isPR)
{
parameters
{
stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form <branch> private BenchviewCommitName')
}
}

// Cap the maximum number of iterations to 21.
parameters {
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
}

def osGroup = getOSGroup(os)
def architecture = 'x64'
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'

steps {
shell("bash ./tests/scripts/perf-prep.sh")
shell("./tests/scripts/perf-prep.sh")
shell("./init-tools.sh")
shell("./build.sh ${architecture} ${configuration}")
shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
"python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\
shell("""./tests/scripts/run-xunit-perf.sh \\
--testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\
--testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\
--coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
--mscorlibDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\
--coreFxBinDir=\"\${WORKSPACE}/corefx\" \\
--runType=\"${runType}\" \\
--benchViewOS=\"${os}\" \\
--generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\
--stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\
--uploadToBenchview""")
}
}

// Save machinedata.json to /artifact/bin/ Jenkins dir
def archiveSettings = new ArchivalSettings()
archiveSettings.addFiles('sandbox/perf-*.xml')
archiveSettings.addFiles('Perf-*.xml')
archiveSettings.addFiles('machinedata.json')
Utilities.addArchival(newJob, archiveSettings)

Expand Down Expand Up @@ -313,14 +324,14 @@ def static getOSGroup(def os) {
[true, false].each { isPR ->
['Ubuntu14.04'].each { os ->
def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {

label('linux_clr_perf')
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
}
}

if (isPR)
{
parameters
Expand All @@ -333,7 +344,7 @@ def static getOSGroup(def os) {
def configuration = 'Release'
def runType = isPR ? 'private' : 'rolling'
def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'

steps {
shell("bash ./tests/scripts/perf-prep.sh --throughput")
shell("./init-tools.sh")
Expand Down
Loading