Skip to content
This repository was archived by the owner on Jan 23, 2023. It is now read-only.

Specify path to python3.5 for perf jobs #20443

Merged
merged 2 commits into from
Nov 8, 2018
Merged

Specify path to python3.5 for perf jobs #20443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db6a22a
Specify path to python3.5 for perf jobs
michellemcdaniel Oct 16, 2018
e8228ba
Change path to python in throughput testing
michellemcdaniel Nov 8, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 33 additions & 26 deletions perf.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,13 @@ def static getOSGroup(def os) {
def architecture = arch
def jobName = isSmoketest ? "perf_perflab_${os}_${arch}_${opt_level}_${jit}_smoketest" : "perf_perflab_${os}_${arch}_${opt_level}_${jit}"
def testEnv = ""
def python = "C:\\Python35\\python.exe"

def newJob = job(Utilities.getFullJobName(project, jobName, isPR)) {
// Set the label.
if (isSmoketest) {
label('Windows.Amd64.ClientRS4.DevEx.15.8.Perf')
python = "C:\\python3.7.0\\python.exe"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Out of curiosity, us there a reason why for the smoke tests we use another version?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I created the helix queue, that was the latest version (and easiest installer to get). When we reinstall the perfsnakes, we'll likely want to upgrade their python install as well.

}
else {
label('windows_server_2016_clr_perf')
Expand Down Expand Up @@ -87,23 +89,23 @@ def static getOSGroup(def os) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")

def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH /AFFINITY 0x2\""

// Run with just stopwatch: Profile=Off
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality")

// Run with the full set of counters enabled: Profile=On
if (opt_level != 'min_opt') {
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi")
}
}
}
Expand Down Expand Up @@ -184,6 +186,8 @@ def static getOSGroup(def os) {
['full_opt', 'min_opt'].each { opt_level ->
def architecture = arch

def python = "C:\\Python35\\python.exe"

pgo_build = ""
pgo_test = ""
pgo_string = "pgo"
Expand Down Expand Up @@ -223,11 +227,11 @@ def static getOSGroup(def os) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}${pgo_build} skiptests")
batchFile("py -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
batchFile("${python} -u tests\\scripts\\run-throughput-perf.py -arch ${arch} -os ${os} -configuration ${configuration} -opt_level ${opt_level} -jit_name ${jit}${pgo_test} -clr_root \"%WORKSPACE%\" -assembly_root \"%WORKSPACE%\\Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os}\\lib\" -benchview_path \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" -run_type ${runType}")
}
}

Expand Down Expand Up @@ -634,6 +638,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
def newJob = job(Utilities.getFullJobName(project, "perf_scenarios_${os}_${arch}_${opt_level}_${jit}", isPR)) {

def testEnv = ""
def python = "C:\\Python35\\python.exe"

// Set the label.
label('windows_server_2016_clr_perf')
Expand Down Expand Up @@ -670,21 +675,21 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")

def runXUnitPerfCommonArgs = "-arch ${arch} -configuration ${configuration} -os ${os} -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} ${testEnv} -optLevel ${opt_level} -jitName ${jit} -outputdir \"%WORKSPACE%\\bin\\sandbox_logs\" -stabilityPrefix \"START \\\"CORECLR_PERF_RUN\\\" /B /WAIT /HIGH\" -scenarioTest"

// Profile=Off
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios")

// Profile=On
if (opt_level != 'min_opt') {
batchFile("py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
batchFile("${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\Scenario\\JitBench -group CoreCLR-Scenarios -collectionFlags BranchMispredictions+CacheMisses+InstructionRetired")
}
}
}
Expand Down Expand Up @@ -767,6 +772,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
def testBin = "%WORKSPACE%\\bin\\tests\\${os}.${architecture}.${configuration}"
def coreRoot = "${testBin}\\Tests\\Core_Root"
def benchViewTools = "%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools"
def python = "C:\\Python35\\python.exe"

steps {
// Install nuget and get BenchView tools
Expand All @@ -780,11 +786,11 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
"${python} \"${benchViewTools}\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"${python} \"${benchViewTools}\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")

// Generate machine data from BenchView
batchFile("py \"${benchViewTools}\\machinedata.py\"")
batchFile("${python} \"${benchViewTools}\\machinedata.py\"")

// Build CoreCLR and gnerate test layout
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")
Expand All @@ -794,11 +800,11 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("\"${coreRoot}\\CoreRun.exe\" \"${testBin}\\sizeondisk\\sodbench\\SoDBench\\SoDBench.exe\" -o \"%WORKSPACE%\\sodbench.csv\" --architecture ${arch} --channel ${channel}")

// From sodbench.csv, create measurment.json, then submission.json
batchFile("py \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
batchFile("py \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")
batchFile("${python} \"${benchViewTools}\\measurement.py\" csv \"%WORKSPACE%\\sodbench.csv\" --metric \"Size on Disk\" --unit \"bytes\" --better \"desc\"")
batchFile("${python} \"${benchViewTools}\\submission.py\" measurement.json --build build.json --machine-data machinedata.json --metadata submission-metadata.json --group \"Dotnet Size on Disk\" --type ${runType} --config-name ${configuration} --architecture ${arch} --machinepool VM --config Channel ${channel}")

// If this is a PR, upload submission.json
batchFile("py \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
batchFile("${python} \"${benchViewTools}\\upload.py\" submission.json --container coreclr")
}
}

Expand Down Expand Up @@ -841,6 +847,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
def newJob = job(Utilities.getFullJobName(project, "perf_illink_${os}_${arch}_${opt_level}_${jit}", isPR)) {

def testEnv = ""
def python = "C:\\Python35\\python.exe"
wrappers {
credentialsBinding {
string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas')
Expand Down Expand Up @@ -874,9 +881,9 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {
batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
"set \"BENCHVIEWNAME=${benchViewName}\"\n" +
"set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=\"\"%\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user-email \"dotnet-bot@microsoft.com\"\n" +
"${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
batchFile("${python} \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
batchFile("set __TestIntermediateDir=int&&build.cmd ${configuration} ${architecture}")

batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly")
Expand All @@ -885,7 +892,7 @@ def static getFullThroughputJobName(def project, def os, def arch, def isPR) {

// Scenario: ILLink
batchFile("\"%VS140COMNTOOLS%\\..\\..\\VC\\vcvarsall.bat\" x86_amd64 && " +
"py tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
"${python} tests\\scripts\\run-xunit-perf.py ${runXUnitPerfCommonArgs} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -group ILLink -nowarmup")
}
}

Expand Down
14 changes: 1 addition & 13 deletions tests/scripts/run-throughput-perf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,17 +98,6 @@
'Ubuntu16.04': 'Linux'
}

python_exe_list = {
'Windows_NT': {
'x64': 'py',
'x86': 'py'
},
'Linux': {
'x64': 'python3.5',
'arm': 'python3.6'
}
}

##########################################################################
# Argument Parser
##########################################################################
Expand Down Expand Up @@ -337,7 +326,6 @@ def main(args):
global dll_exclude_list
global jit_list
global os_group_list
global python_exe_list

architecture, operating_system, os_group, build_type, run_type, clr_root, assembly_root, benchview_path, iterations, opt_level, jit_name, no_pgo = validate_args(args)
arch = architecture
Expand Down Expand Up @@ -371,7 +359,7 @@ def main(args):
# Replace assembly_root's System.Private.CoreLib with built System.Private.CoreLib.
shutil.copyfile(os.path.join(bin_path, 'System.Private.CoreLib.dll'), os.path.join(assembly_root, 'System.Private.CoreLib.dll'))

python_exe = python_exe_list[os_group][arch]
python_exe = sys.executable

# Run throughput testing
for dll_file_name in os.listdir(assembly_root):
Expand Down
4 changes: 1 addition & 3 deletions tests/scripts/run-xunit-perf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,13 +418,11 @@ def main(args):
coreclrPerf, assemblyName, arch, operatingSystem, configuration, jitName, optLevel, runType, outputDir, stabilityPrefix, isScenarioTest, benchviewPath, isPgoOptimized, benchviewGroup, hasWarmupRun, collectionFlags, isLibrary, uploadToBenchview, better, sliceNumber, sliceConfigFile = validate_args(args)

platform = sys.platform
python = 'py'
python = sys.executable
if platform == 'linux' or platform == 'linux2':
platform = 'Linux'
python = 'python3'
elif platform == 'darwin':
platform = 'OSX'
python = 'python3'
elif platform == 'win32':
platform = "Windows_NT"
else:
Expand Down