Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Commit 5378afc

Browse files
michellemcdanielmichellemcdaniel
authored
Convert run-xunit-perf to py script (#15568)
* Convert run-xunit-perf to python script This change merges the two run-xunit-perf scripts (.sh and .cmd) into one unified python script and updates the pipeline job to use the pythong script. This change also updates the linux jobs to use the new build-tests.sh generatelayoutonly command so that we don't need to pull down corefx from the cloud anymore. The unified python script enables us to more easily update both linux scripting and windows scripting at the same time so that one does not lag behind the other (such as when we add new configurations or options like slicing). This change also turns linux testing back on by default for PRs.
1 parent 3d6da79 commit 5378afc

File tree

4 files changed

+670
-79
lines changed

4 files changed

+670
-79
lines changed

‎buildpipeline/perf-pipeline.groovy‎

Lines changed: 39 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,6 @@
44
// Note that the parameters will be set as env variables so we cannot use names that conflict
55
// with the engineering system parameter names.
66

7-
//-------------------------- Globals ---------------------------------//
8-
9-
def validTestFolders = [
10-
'Benchstones',
11-
'BenchmarksGame',
12-
'Bytemark',
13-
'Math',
14-
'Span',
15-
'first half',
16-
'last half'
17-
]
18-
197
//--------------------- Windows Functions ----------------------------//
208

219
def windowsBuild(String arch, String config, String pgo, boolean isBaseline) {
@@ -39,7 +27,7 @@ def windowsBuild(String arch, String config, String pgo, boolean isBaseline) {
3927
stash name: "nt-${arch}-${pgo}${baselineString}-test-artifacts", includes: 'bin/tests/**'
4028
}
4129

42-
def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn, String testFolder) {
30+
def windowsPerf(String arch, String config, String uploadString, String runType, String opt_level, String jit, String pgo, String scenario, boolean isBaseline, boolean isProfileOn, int slice) {
4331
withCredentials([string(credentialsId: 'CoreCLR Perf BenchView Sas', variable: 'BV_UPLOAD_SAS_TOKEN')]) {
4432
checkout scm
4533
String baselineString = ""
@@ -52,11 +40,6 @@ def windowsPerf(String arch, String config, String uploadString, String runType,
5240
unstash "metadata"
5341
}
5442

55-
String test = ''
56-
if (testFolder != 'all') {
57-
test = testFolder
58-
}
59-
6043
String pgoTestFlag = ((pgo == 'nopgo') ? '-nopgo' : '')
6144

6245
// We want to use the baseline metadata for baseline runs. We expect to find the submission metadata in
@@ -78,56 +61,34 @@ def windowsPerf(String arch, String config, String uploadString, String runType,
7861

7962
String runXUnitCommonArgs = "-arch ${arch} -configuration ${config} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} ${pgoTestFlag} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\""
8063
if (scenario == 'perf') {
81-
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\""
82-
if (test == 'first half' || testFolder == 'all')
64+
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""
65+
if (slice == -1)
8366
{
8467
String runXUnitPerflabArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\perflab\\Perflab -library"
8568

8669
profileArg = isProfileOn ? "default+${profileArg}+gcapi" : profileArg
87-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
88-
}
70+
bat "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerflabArgs} -collectionFlags ${profileArg}"
8971

90-
if (test == 'first half') {
91-
92-
[
93-
'Burgers',
94-
'Devirtualization',
95-
'FractalPerf',
96-
'Inlining',
97-
'Layout'
98-
].each { benchmark ->
99-
String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
100-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
101-
}
102-
}
103-
else if (test == 'last half') {
104-
[
105-
'Linq',
106-
'Roslyn',
107-
'SciMark',
108-
'Serialization',
109-
'V8'
110-
].each { benchmark ->
111-
String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${benchmark}"
112-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
113-
}
72+
String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\"
73+
bat "py tests\\scripts\\run-xunit-perf.py ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
11474
}
75+
11576
else {
116-
String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\Jit\\Performance\\CodeQuality\\${test}"
117-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
77+
String runXUnitCodeQualityArgs = "${runXUnitPerfCommonArgs} -slice ${slice} -sliceConfigFile \"%WORKSPACE%\\tests\\scripts\\perf-slices.json\" -testBinLoc bin\\tests\\${os}.${arch}.${config}"
78+
bat "py tests\\scripts\\run-xunit-perf.py ${runXUnitCodeQualityArgs} -collectionFlags ${profileArg}"
11879
}
11980
}
12081
else if (scenario == 'jitbench') {
12182
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH\" -scenarioTest"
12283
runXUnitPerfCommonArgs = "${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios"
12384

12485
if (!(opt_level == 'min_opt' && isProfileOn)) {
125-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -collectionFlags ${profileArgs}"
86+
bat "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -collectionFlags ${profileArgs}"
12687
}
12788
}
12889
else if (scenario == 'illink') {
12990
String runXUnitPerfCommonArgs = "${runXUnitCommonArgs} -scenarioTest"
130-
bat "tests\\scripts\\run-xunit-perf.cmd ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\linkbench\\linkbench -group ILLink -nowarmup"
91+
bat "py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${arch}.${config}\\performance\\linkbench\\linkbench -group ILLink -nowarmup"
13192
}
13293
archiveArtifacts allowEmptyArchive: false, artifacts:'bin/sandbox_logs/**,machinedata.json'
13394
}
@@ -192,7 +153,7 @@ def linuxPerf(String arch, String os, String config, String uploadString, String
192153
baselineString = "-baseline"
193154
}
194155

195-
String pgoTestFlag = ((pgo == 'nopgo') ? '--nopgo' : '')
156+
String pgoTestFlag = ((pgo == 'nopgo') ? '-nopgo' : '')
196157

197158
dir ('.') {
198159
unstash "linux-${arch}-${pgo}${baselineString}-build-artifacts"
@@ -209,9 +170,13 @@ def linuxPerf(String arch, String os, String config, String uploadString, String
209170
sh "mv -f submission-metadata-baseline.json submission-metadata.json"
210171
}
211172

212-
sh "./tests/scripts/perf-prep.sh"
173+
sh "./tests/scripts/perf-prep.sh --nocorefx"
213174
sh "./init-tools.sh"
214-
sh "./tests/scripts/run-xunit-perf.sh --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${arch}.${config}\" --optLevel=${optLevel} ${pgoTestFlag} --testNativeBinDir=\"\${WORKSPACE}/bin/obj/Linux.${arch}.${config}/tests\" --coreClrBinDir=\"\${WORKSPACE}/bin/Product/Linux.${arch}.${config}\" --mscorlibDir=\"\${WORKSPACE}/bin/Product/Linux.${arch}.${config}\" --coreFxBinDir=\"\${WORKSPACE}/corefx\" --runType=\"${runType}\" --benchViewOS=\"${os}\" --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" --uploadToBenchview --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\""
175+
sh "./build-test.sh release $arch generatelayoutonly"
176+
177+
String runXUnitCommonArgs = "-arch ${arch} -os Ubuntu16.04 -configuration ${config} -stabilityPrefix \"taskset 0x00000002 nice --adjustment=-10\" -generateBenchviewData \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" ${uploadString} ${pgoTestFlag} -runtype ${runType} -optLevel ${optLevel} -outputdir \"\${WORKSPACE}/bin/sandbox_logs\""
178+
179+
sh "python3 ./tests/scripts/run-xunit-perf.py -testBinLoc bin/tests/Windows_NT.${arch}.${config}/JIT/Performance/CodeQuality ${runXUnitCommonArgs}"
215180
archiveArtifacts allowEmptyArchive: false, artifacts:'bin/toArchive/**,machinedata.json'
216181
}
217182
}
@@ -294,6 +259,11 @@ def innerLoopBuilds = [
294259
simpleNode('Windows_NT','latest') {
295260
windowsBuild('x86', config, 'pgo', false)
296261
}
262+
},
263+
"linux x64 pgo build": {
264+
simpleNode('RHEL7.2', 'latest-or-auto') {
265+
linuxBuild('x64', config, 'pgo', false)
266+
}
297267
}
298268
]
299269

@@ -312,11 +282,6 @@ if (!isPR()) {
312282
windowsBuild('x86', config, 'nopgo', false)
313283
}
314284
},
315-
"linux x64 pgo build": {
316-
simpleNode('RHEL7.2', 'latest-or-auto') {
317-
linuxBuild('x64', config, 'pgo', false)
318-
}
319-
},
320285
"linux x64 nopgo build": {
321286
simpleNode('RHEL7.2', 'latest-or-auto') {
322287
linuxBuild('x64', config, 'nopgo', false)
@@ -354,17 +319,26 @@ def innerLoopTests = [:]
354319
['x64', 'x86'].each { arch ->
355320
['full_opt'].each { opt_level ->
356321
[false].each { isBaseline ->
357-
validTestFolders.each { benchmark ->
322+
[0,1,2,3,4,5].each { slice ->
358323
String baseline = ""
359324
if (isBaseline) {
360325
baseline = " baseline"
361326
}
362327
if (isPR() || !isBaseline) {
363-
innerLoopTests["windows ${arch} ryujit ${opt_level} pgo ${benchmark}${baseline} perf"] = {
328+
innerLoopTests["windows ${arch} ryujit ${opt_level} pgo ${slice}${baseline} perf"] = {
364329
simpleNode('windows_server_2016_clr_perf', 180) {
365-
windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true, benchmark)
330+
windowsPerf(arch, config, uploadString, runType, opt_level, 'ryujit', 'pgo', 'perf', isBaseline, true, slice)
366331
}
367332
}
333+
334+
}
335+
}
336+
337+
if (arch == 'x64') {
338+
innerLoopTests["linux ${arch} ryujit ${opt_level} pgo perf"] = {
339+
simpleNode('ubuntu_1604_clr_perf', 180) {
340+
linuxPerf(arch, 'Ubuntu16.04', config, uploadString, runType, opt_level, 'pgo', false)
341+
}
368342
}
369343
}
370344
}
@@ -376,15 +350,15 @@ def outerLoopTests = [:]
376350

377351
if (!isPR()) {
378352
['x64', 'x86'].each { arch ->
379-
outerLoopTests["windows ${arch} ryujit full_opt pgo jitbench"] = {
353+
outerLoopTests["windows ${arch} ryujit full_opt pgo${baseline} jitbench"] = {
380354
simpleNode('windows_server_2016_clr_perf', 180) {
381-
windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false, false, '')
355+
windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'jitbench', false, false, -1)
382356
}
383357
}
384358

385359
outerLoopTests["windows ${arch} ryujit full_opt pgo illink"] = {
386360
simpleNode('Windows_NT', '20170427-elevated') {
387-
windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false, false, '')
361+
windowsPerf(arch, config, uploadString, runType, 'full_opt', 'ryujit', 'pgo', 'illink', false, false, -1)
388362
}
389363
}
390364
}
@@ -396,7 +370,7 @@ if (!isPR()) {
396370
[true, false].each { isProfileOn ->
397371
outerLoopTests["windows ${arch} ${jit} ${opt_level} ${pgo_enabled} perf"] = {
398372
simpleNode('windows_server_2016_clr_perf', 180) {
399-
windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn, 'all')
373+
windowsPerf(arch, config, uploadString, runType, opt_level, jit, pgo_enabled, 'perf', false, isProfileOn, -1)
400374
}
401375
}
402376

‎tests/scripts/perf-prep.sh‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ perfArch="x64"
2323
perfConfig="Release"
2424
perfBranch=
2525
throughput=0
26+
nocorefx=0
2627

2728
for i in "$@"
2829
do
@@ -37,6 +38,9 @@ do
3738
-t|--throughput)
3839
throughput=1
3940
;;
41+
--nocorefx)
42+
nocorefx=1
43+
;;
4044
*)
4145
echo "Unknown switch: $i"
4246
print_usage
@@ -63,8 +67,8 @@ if [ ! -d "./tests/scripts/Microsoft.Benchview.JSONFormat" ]; then
6367
fi
6468

6569
# Install python 3.5.2 to run machinedata.py for machine data collection
66-
python3.5 --version
67-
python3.5 ./tests/scripts/Microsoft.BenchView.JSONFormat/tools/machinedata.py
70+
python3 --version
71+
python3 ./tests/scripts/Microsoft.BenchView.JSONFormat/tools/machinedata.py
6872

6973
if [ $throughput -eq 1 ]; then
7074
# Download throughput benchmarks
@@ -78,27 +82,33 @@ if [ $throughput -eq 1 ]; then
7882
fi
7983

8084
else
81-
# Set up the copies
82-
# Coreclr build containing the tests and mscorlib
83-
curl https://ci.dot.net/job/$perfBranch/job/master/job/release_windows_nt/lastSuccessfulBuild/artifact/bin/tests/tests.zip -o tests.zip
84-
85-
# Corefx components. We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
86-
mkdir corefx
87-
curl https://ci.dot.net/job/dotnet_corefx/job/master/job/ubuntu14.04_release/lastSuccessfulBuild/artifact/bin/build.tar.gz -o ./corefx/build.tar.gz
85+
if [ $nocorefx -eq 0 ]; then
86+
# Corefx components. We now have full stack builds on all distros we test here, so we can copy straight from CoreFX jobs.
87+
echo "Downloading corefx"
88+
mkdir corefx
89+
curl https://ci.dot.net/job/dotnet_corefx/job/master/job/ubuntu14.04_release/lastSuccessfulBuild/artifact/bin/build.tar.gz -o ./corefx/build.tar.gz
8890

89-
# Unpack the corefx binaries
90-
pushd corefx > /dev/null
91-
tar -xf build.tar.gz
92-
rm build.tar.gz
93-
popd > /dev/null
91+
# Unpack the corefx binaries
92+
pushd corefx > /dev/null
93+
tar -xf build.tar.gz
94+
rm build.tar.gz
95+
popd > /dev/null
96+
fi
9497

9598
# If the tests don't already exist, download them.
9699
if [ ! -d "bin" ]; then
100+
echo "Making bin dir"
97101
mkdir bin
98102
fi
99103

100104
if [ ! -d "bin/tests" ]; then
105+
echo "Making bin/tests"
101106
mkdir bin/tests
107+
fi
108+
109+
if [ ! -d "bin/tests/Windows_NT.$perfArch.$perfConfig" ]; then
110+
echo "Downloading tests"
111+
curl https://ci.dot.net/job/$perfBranch/job/master/job/release_windows_nt/lastSuccessfulBuild/artifact/bin/tests/tests.zip -o tests.zip
102112
echo "unzip tests to ./bin/tests/Windows_NT.$perfArch.$perfConfig"
103113
unzip -q -o tests.zip -d ./bin/tests/Windows_NT.$perfArch.$perfConfig || exit 0
104114
fi

‎tests/scripts/perf-slices.json‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
{
2+
"slices" : [
3+
{
4+
"folders" : [
5+
{
6+
"directory" : "JIT/Performance/CodeQuality/Benchstones",
7+
"extraFlags" : ""
8+
}
9+
]
10+
},
11+
{
12+
"folders" : [
13+
{
14+
"directory" : "JIT/Performance/CodeQuality/BenchmarksGame",
15+
"extraFlags" : ""
16+
}
17+
]
18+
},
19+
{
20+
"folders" : [
21+
{
22+
"directory" : "JIT/Performance/CodeQuality/Bytemark",
23+
"extraFlags" : ""
24+
}
25+
]
26+
},
27+
{
28+
"folders" : [
29+
{
30+
"directory" : "JIT/Performance/CodeQuality/Span",
31+
"extraFlags" : ""
32+
}
33+
]
34+
},
35+
{
36+
"folders" : [
37+
{
38+
"directory" : "performance/perflab",
39+
"extraFlags" : "-library"
40+
},
41+
{
42+
"directory" : "JIT/Performance/CodeQuality/Burgers",
43+
"extraFlags" : ""
44+
},
45+
{
46+
"directory" : "JIT/Performance/CodeQuality/Devirtualization",
47+
"extraFlags" : ""
48+
},
49+
{
50+
"directory" : "JIT/Performance/CodeQuality/FractalPerf",
51+
"extraFlags" : ""
52+
},
53+
{
54+
"directory" : "JIT/Performance/CodeQuality/Inlining",
55+
"extraFlags" : ""
56+
},
57+
{
58+
"directory" : "JIT/Performance/CodeQuality/Layout",
59+
"extraFlags" : ""
60+
}
61+
]
62+
},
63+
{
64+
"folders" : [
65+
{
66+
"directory" : "JIT/Performance/CodeQuality/Linq",
67+
"extraFlags" : ""
68+
},
69+
{
70+
"directory" : "JIT/Performance/CodeQuality/Roslyn",
71+
"extraFlags" : ""
72+
},
73+
{
74+
"directory" : "JIT/Performance/CodeQuality/SciMark",
75+
"extraFlags" : ""
76+
},
77+
{
78+
"directory" : "JIT/Performance/CodeQuality/Serialization",
79+
"extraFlags" : ""
80+
},
81+
{
82+
"directory" : "JIT/Performance/CodeQuality/V8",
83+
"extraFlags" : ""
84+
}
85+
]
86+
}
87+
]
88+
}

0 commit comments

Comments
 (0)