From 369f1c9cb877eb55bf1a267e7e80fe359bd14da2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Fri, 26 May 2017 18:01:05 -0700 Subject: [PATCH 1/2] Enable linux perf runs (#11905) - Removing `[assembly: MeasureInstructionsRetired]` from performance tests. - Due to this issue: https://github.com/Microsoft/xunit-performance/issues/231, we are blocked from running on performance tests on Linux. The tag is not used by infrastructure and it is passed during runtime when profiling on Windows. - Adding the option to execute CoreRun with the stability prefix. - Moving file to be archived by Jenkins and updating groovy file. - Setting the maximum number of iterations. --- perf.groovy | 89 ++--- tests/scripts/run-xunit-perf.sh | 312 ++++++++---------- .../src/Common/PerfHarness/PerfHarness.csproj | 2 +- tests/src/Common/external/external.depproj | 8 +- .../CodeQuality/BenchF/Adams/Adams.cs | 1 - .../CodeQuality/BenchF/BenchMk2/BenchMk2.cs | 1 - .../CodeQuality/BenchF/BenchMrk/BenchMrk.cs | 1 - .../CodeQuality/BenchF/Bisect/Bisect.cs | 1 - .../CodeQuality/BenchF/DMath/DMath.cs | 1 - .../Performance/CodeQuality/BenchF/FFT/FFT.cs | 1 - .../CodeQuality/BenchF/InProd/InProd.cs | 1 - .../CodeQuality/BenchF/InvMt/InvMt.cs | 1 - .../CodeQuality/BenchF/LLoops/LLoops.cs | 1 - .../CodeQuality/BenchF/Lorenz/Lorenz.cs | 1 - .../CodeQuality/BenchF/MatInv4/MatInv4.cs | 1 - .../CodeQuality/BenchF/NewtE/NewtE.cs | 1 - .../CodeQuality/BenchF/NewtR/NewtR.cs | 1 - .../CodeQuality/BenchF/Regula/Regula.cs | 1 - .../CodeQuality/BenchF/Romber/Romber.cs | 1 - .../CodeQuality/BenchF/Secant/Secant.cs | 1 - .../CodeQuality/BenchF/Simpsn/Simpsn.cs | 1 - .../CodeQuality/BenchF/SqMtx/SqMtx.cs | 1 - .../CodeQuality/BenchF/Trap/Trap.cs | 1 - .../CodeQuality/BenchF/Whetsto/Whetsto.cs | 1 - .../CodeQuality/BenchI/8Queens/8Queens.cs | 1 - .../CodeQuality/BenchI/Ackermann/Ackermann.cs | 1 - .../CodeQuality/BenchI/AddArray/AddArray.cs | 1 - .../CodeQuality/BenchI/AddArray2/AddArray2.cs | 1 - .../CodeQuality/BenchI/Array1/Array1.cs | 1 - .../CodeQuality/BenchI/Array2/Array2.cs | 1 - .../CodeQuality/BenchI/BenchE/BenchE.cs | 1 - .../BenchI/BubbleSort/BubbleSort.cs | 1 - .../BenchI/BubbleSort2/BubbleSort2.cs | 1 - .../CodeQuality/BenchI/CSieve/CSieve.cs | 1 - .../Performance/CodeQuality/BenchI/Fib/Fib.cs | 1 - .../CodeQuality/BenchI/HeapSort/HeapSort.cs | 1 - .../CodeQuality/BenchI/IniArray/IniArray.cs | 1 - .../BenchI/LogicArray/LogicArray.cs | 1 - .../CodeQuality/BenchI/Midpoint/Midpoint.cs | 1 - .../CodeQuality/BenchI/MulMatrix/MulMatrix.cs | 1 - .../BenchI/NDhrystone/NDhrystone.cs | 1 - .../CodeQuality/BenchI/Permutate/Permutate.cs | 1 - .../Performance/CodeQuality/BenchI/Pi/Pi.cs | 1 - .../CodeQuality/BenchI/Puzzle/Puzzle.cs | 1 - .../CodeQuality/BenchI/QuickSort/QuickSort.cs | 1 - .../BenchI/TreeInsert/TreeInsert.cs | 1 - .../CodeQuality/BenchI/TreeSort/TreeSort.cs | 1 - .../BenchI/XposMatrix/XposMatrix.cs | 1 - .../binarytrees/binarytrees.csharp.cs | 1 - .../binarytrees/binarytrees.csharp3.cs | 1 - .../BenchmarksGame/fasta/fasta.csharp-2.cs | 1 - .../fastaredux/fastaredux.csharp.cs | 1 - .../k-nucleotide/k-nucleotide.cs | 13 +- .../BenchmarksGame/nbody/nbody.csharp-3.cs | 1 - .../BenchmarksGame/pidigits/pi-digits.cs | 1 - .../regexdna/regexdna.csharp-6.cs | 1 - .../revcomp/revcomp.csharp-1.cs | 1 - .../spectralnorm/spectralnorm.cs | 1 - .../CodeQuality/Burgers/Burgers.cs | 5 +- .../CodeQuality/Bytemark/ByteMark.cs | 31 +- .../CodeQuality/FractalPerf/FractalPerf.cs | 11 +- .../CodeQuality/Inlining/InlineGCStruct.cs | 1 - .../CodeQuality/Inlining/NoThrowInline.cs | 1 - .../JIT/Performance/CodeQuality/Linq/Linq.cs | 1 - .../CodeQuality/Math/Functions/Functions.cs | 1 - .../CodeQuality/Roslyn/CscBench.cs | 1 - .../SIMD/ConsoleMandel/ConsoleMandel.cs | 1 - .../SIMD/RayTracer/RayTracerBench.cs | 1 - .../CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs | 1 - .../Performance/CodeQuality/SciMark/kernel.cs | 7 +- .../CodeQuality/Serialization/Deserialize.cs | 1 - .../CodeQuality/Serialization/Serialize.cs | 1 - .../Performance/CodeQuality/Span/Indexer.cs | 5 +- .../Performance/CodeQuality/Span/SpanBench.cs | 7 +- .../CodeQuality/V8/Crypto/Crypto.cs | 1 - .../CodeQuality/V8/Richards/Richards.cs | 1 - .../benchmark+roslyn/benchmark+roslyn.csproj | 8 +- .../benchmark+serialize.csproj | 8 +- tests/src/performance/performance.csproj | 8 +- 79 files changed, 248 insertions(+), 331 deletions(-) diff --git a/perf.groovy b/perf.groovy index 2674faf06e0f..125ed7b06218 100644 --- a/perf.groovy +++ b/perf.groovy @@ -24,6 +24,7 @@ def static getOSGroup(def os) { assert osGroup != null : "Could not find os group for ${os}" return osGroupMap[os] } + // Setup perflab tests runs [true, false].each { isPR -> ['Windows_NT'].each { os -> @@ -51,34 +52,34 @@ def static getOSGroup(def os) { } } - if (isPR) - { - parameters + if (isPR) { - stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form private BenchviewCommitName') + parameters + { + stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form private BenchviewCommitName') + } } - } - if (isSmoketest) - { - parameters + if (isSmoketest) { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') + parameters + { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '2', 'Sets the number of iterations to two. We want to do this so that we can run as fast as possible as this is just for smoke testing') + } } - } - else - { - parameters + else { - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') - stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + parameters + { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + } } - } - def configuration = 'Release' - def runType = isPR ? 'private' : 'rolling' - def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - def uploadString = isSmoketest ? '' : '-uploadToBenchview' - + def configuration = 'Release' + def runType = isPR ? 'private' : 'rolling' + def benchViewName = isPR ? 'coreclr private %BenchviewCommitName%' : 'coreclr rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' + def uploadString = isSmoketest ? '' : '-uploadToBenchview' + steps { // Batch @@ -114,7 +115,7 @@ def static getOSGroup(def os) { batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") } } - + if (isSmoketest) { Utilities.setMachineAffinity(newJob, "Windows_NT", '20170427-elevated') @@ -123,10 +124,11 @@ def static getOSGroup(def os) { def archiveSettings = new ArchivalSettings() archiveSettings.addFiles('Perf-*.xml') archiveSettings.addFiles('Perf-*.etl') + archiveSettings.addFiles('machinedata.json') Utilities.addArchival(newJob, archiveSettings) Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}") - + newJob.with { wrappers { timeout { @@ -134,7 +136,7 @@ def static getOSGroup(def os) { } } } - + if (isPR) { TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest() if (isSmoketest) @@ -185,7 +187,7 @@ def static getOSGroup(def os) { def configuration = 'Release' def runType = isPR ? 'private' : 'rolling' def benchViewName = isPR ? 'coreclr-throughput private %BenchviewCommitName%' : 'coreclr-throughput rolling %GIT_BRANCH_WITHOUT_ORIGIN% %GIT_COMMIT%' - + steps { // Batch @@ -235,14 +237,14 @@ def static getOSGroup(def os) { [true, false].each { isPR -> ['Ubuntu14.04'].each { os -> def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) { - + label('linux_clr_perf') - wrappers { - credentialsBinding { - string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') - } + wrappers { + credentialsBinding { + string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') } - + } + if (isPR) { parameters @@ -250,20 +252,27 @@ def static getOSGroup(def os) { stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview. The final name will be of the form private BenchviewCommitName') } } + + // Cap the maximum number of iterations to 21. + parameters { + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one. We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample') + } + def osGroup = getOSGroup(os) def architecture = 'x64' def configuration = 'Release' def runType = isPR ? 'private' : 'rolling' def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' - + steps { - shell("bash ./tests/scripts/perf-prep.sh") + shell("./tests/scripts/perf-prep.sh") shell("./init-tools.sh") shell("./build.sh ${architecture} ${configuration}") shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" + "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}") - shell("""sudo -E bash ./tests/scripts/run-xunit-perf.sh \\ + shell("""./tests/scripts/run-xunit-perf.sh \\ --testRootDir=\"\${WORKSPACE}/bin/tests/Windows_NT.${architecture}.${configuration}\" \\ --testNativeBinDir=\"\${WORKSPACE}/bin/obj/${osGroup}.${architecture}.${configuration}/tests\" \\ --coreClrBinDir=\"\${WORKSPACE}/bin/Product/${osGroup}.${architecture}.${configuration}\" \\ @@ -271,13 +280,15 @@ def static getOSGroup(def os) { --coreFxBinDir=\"\${WORKSPACE}/corefx\" \\ --runType=\"${runType}\" \\ --benchViewOS=\"${os}\" \\ + --generatebenchviewdata=\"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools\" \\ + --stabilityPrefix=\"taskset 0x00000002 nice --adjustment=-10\" \\ --uploadToBenchview""") } } // Save machinedata.json to /artifact/bin/ Jenkins dir def archiveSettings = new ArchivalSettings() - archiveSettings.addFiles('sandbox/perf-*.xml') + archiveSettings.addFiles('Perf-*.xml') archiveSettings.addFiles('machinedata.json') Utilities.addArchival(newJob, archiveSettings) @@ -313,14 +324,14 @@ def static getOSGroup(def os) { [true, false].each { isPR -> ['Ubuntu14.04'].each { os -> def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) { - + label('linux_clr_perf') wrappers { credentialsBinding { string('BV_UPLOAD_SAS_TOKEN', 'CoreCLR Perf BenchView Sas') } } - + if (isPR) { parameters @@ -333,7 +344,7 @@ def static getOSGroup(def os) { def configuration = 'Release' def runType = isPR ? 'private' : 'rolling' def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT' - + steps { shell("bash ./tests/scripts/perf-prep.sh --throughput") shell("./init-tools.sh") diff --git a/tests/scripts/run-xunit-perf.sh b/tests/scripts/run-xunit-perf.sh index 6f49bf6e0aa6..2797174f329c 100755 --- a/tests/scripts/run-xunit-perf.sh +++ b/tests/scripts/run-xunit-perf.sh @@ -1,5 +1,13 @@ #!/usr/bin/env bash +function run_command { + echo "" + echo $USER@`hostname` "$PWD" + echo `date +"[%m/%d/%Y %H:%M:%S]"`" $ $@" + "$@" + return $? +} + function print_usage { echo '' echo 'CoreCLR perf test script on Linux.' @@ -14,29 +22,26 @@ function print_usage { echo ' --coreFxBinDir="corefx/bin/Linux.AnyCPU.Debug"' echo '' echo 'Required arguments:' - echo ' --testRootDir= : Root directory of the test build (e.g. coreclr/bin/tests/Windows_NT.x64.Debug).' - echo ' --testNativeBinDir= : Directory of the native CoreCLR test build (e.g. coreclr/bin/obj/Linux.x64.Debug/tests).' + echo ' --testRootDir= : Root directory of the test build (e.g. coreclr/bin/tests/Windows_NT.x64.Debug).' + echo ' --testNativeBinDir= : Directory of the native CoreCLR test build (e.g. coreclr/bin/obj/Linux.x64.Debug/tests).' echo ' (Also required: Either --coreOverlayDir, or all of the switches --coreOverlayDir overrides)' echo '' echo 'Optional arguments:' - echo ' --coreOverlayDir= : Directory containing core binaries and test dependencies. If not specified, the' - echo ' default is testRootDir/Tests/coreoverlay. This switch overrides --coreClrBinDir,' - echo ' --mscorlibDir, and --coreFxBinDir.' - echo ' --coreClrBinDir= : Directory of the CoreCLR build (e.g. coreclr/bin/Product/Linux.x64.Debug).' - echo ' --mscorlibDir= : Directory containing the built mscorlib.dll. If not specified, it is expected to be' + echo ' --coreOverlayDir= : Directory containing core binaries and test dependencies. If not specified, the' + echo ' default is testRootDir/Tests/coreoverlay. This switch overrides --coreClrBinDir,' + echo ' --mscorlibDir, and --coreFxBinDir.' + echo ' --coreClrBinDir= : Directory of the CoreCLR build (e.g. coreclr/bin/Product/Linux.x64.Debug).' + echo ' --mscorlibDir= : Directory containing the built mscorlib.dll. If not specified, it is expected to be' echo ' in the directory specified by --coreClrBinDir.' - echo ' --coreFxBinDir="" : The path to the unpacked runtime folder that is produced as part of a CoreFX build' - echo ' --uploadToBenchview : Specify this flag in order to have the results of the run uploaded to Benchview.' - echo ' This also requires that the os flag and runtype flag to be set. Lastly you must' - echo ' also have the BV_UPLOAD_SAS_TOKEN set to a SAS token for the Benchview upload container' - echo ' --benchViewOS= : Specify the os that will be used to insert data into Benchview.' - echo ' --runType= : Specify the runType for Benchview.' + echo ' --coreFxBinDir="" : The path to the unpacked runtime folder that is produced as part of a CoreFX build' + echo ' --generatebenchviewdata : BenchView tools directory.' + echo ' --uploadToBenchview : Specify this flag in order to have the results of the run uploaded to Benchview.' + echo ' This requires that the generatebenchviewdata, os and runtype flags to be set, and' + echo ' also have the BV_UPLOAD_SAS_TOKEN set to a SAS token for the Benchview upload container' + echo ' --benchViewOS= : Specify the os that will be used to insert data into Benchview.' + echo ' --runType= : Specify the runType for Benchview. [Default: local]' } -# Variables for xUnit-style XML output. XML format: https://xunit.github.io/docs/format-xml-v2.html -xunitOutputPath= -xunitTestOutputPath= - # libExtension determines extension for dynamic library files OSName=$(uname -s) libExtension= @@ -59,95 +64,6 @@ case $OSName in ;; esac -function xunit_output_end { - local errorSource=$1 - local errorMessage=$2 - - local errorCount - if [ -z "$errorSource" ]; then - ((errorCount = 0)) - else - ((errorCount = 1)) - fi - - echo '' >>"$xunitOutputPath" - echo '' >>"$xunitOutputPath" - - local line - - # - line=" " - line="${line}>"$xunitOutputPath" - - # - line=" " - line="${line}>"$xunitOutputPath" - - # ... - if [ -f "$xunitTestOutputPath" ]; then - cat "$xunitTestOutputPath" >>"$xunitOutputPath" - rm -f "$xunitTestOutputPath" - fi - - # - line=" " - line="${line}" - echo "$line" >>"$xunitOutputPath" - - if [ -n "$errorSource" ]; then - # - line=" " - line="${line}" - echo "$line" >>"$xunitOutputPath" - - # - line=" " - line="${line}>"$xunitOutputPath" - - # - line=" " - line="${line}${errorMessage}" - echo "$line" >>"$xunitOutputPath" - - # - line=" " - line="${line}" - echo "$line" >>"$xunitOutputPath" - - # - line=" " - line="${line}" - echo "$line" >>"$xunitOutputPath" - fi - - # - line=" " - line="${line}" - echo "$line" >>"$xunitOutputPath" - - # - echo '' >>"$xunitOutputPath" -} - function exit_with_error { local errorSource=$1 local errorMessage=$2 @@ -158,10 +74,11 @@ function exit_with_error { fi echo "$errorMessage" - xunit_output_end "$errorSource" "$errorMessage" if ((printUsage != 0)); then print_usage fi + + echo "Exiting script with error code: $EXIT_CODE_EXCEPTION" exit $EXIT_CODE_EXCEPTION } @@ -185,11 +102,12 @@ function create_core_overlay { if [ -n "$coreOverlayDir" ]; then export CORE_ROOT="$coreOverlayDir" - return + return 0 fi - # Check inputs to make sure we have enough information to create the core layout. $testRootDir/Tests/Core_Root should - # already exist and contain test dependencies that are not built. + # Check inputs to make sure we have enough information to create the core + # layout. $testRootDir/Tests/Core_Root should already exist and contain test + # dependencies that are not built. local testDependenciesDir=$testRootDir/Tests/Core_Root if [ ! -d "$testDependenciesDir" ]; then exit_with_error "$errorSource" "Did not find the test dependencies directory: $testDependenciesDir" @@ -208,13 +126,30 @@ function create_core_overlay { coreOverlayDir=$testRootDir/Tests/coreoverlay export CORE_ROOT="$coreOverlayDir" if [ -e "$coreOverlayDir" ]; then - rm -f -r "$coreOverlayDir" + rm -rf "$coreOverlayDir" || exit 1 fi + mkdir "$coreOverlayDir" - cp -f -v "$coreFxBinDir"/* "$coreOverlayDir/" 2>/dev/null - cp -f -v "$coreClrBinDir/"* "$coreOverlayDir/" 2>/dev/null - cp -n -v "$testDependenciesDir"/* "$coreOverlayDir/" 2>/dev/null + cp -f -v "$coreFxBinDir/"* "$coreOverlayDir/" || exit 2 + cp -f -p -v "$coreClrBinDir/"* "$coreOverlayDir/" # || exit 3 + if [ -d "$mscorlibDir/bin" ]; then + cp -f -v "$mscorlibDir/bin/"* "$coreOverlayDir/" || exit 4 + fi + cp -f -v "$testDependenciesDir/"xunit* "$coreOverlayDir/" || exit 5 + cp -n -v "$testDependenciesDir/"* "$coreOverlayDir/" # || exit 6 + if [ -f "$coreOverlayDir/mscorlib.ni.dll" ]; then + # Test dependencies come from a Windows build, and mscorlib.ni.dll would be the one from Windows + rm -f "$coreOverlayDir/mscorlib.ni.dll" || exit 7 + fi + if [ -f "$coreOverlayDir/System.Private.CoreLib.ni.dll" ]; then + # Test dependencies come from a Windows build, and System.Private.CoreLib.ni.dll would be the one from Windows + rm -f "$coreOverlayDir/System.Private.CoreLib.ni.dll" || exit 8 + fi + + copy_test_native_bin_to_test_root || exit 9 + + return 0 } function precompile_overlay_assemblies { @@ -227,19 +162,19 @@ function precompile_overlay_assemblies { for fileToPrecompile in ${filesToPrecompile} do local filename=${fileToPrecompile} - echo Precompiling $filename + echo "Precompiling $filename" $overlayDir/crossgen /Platform_Assemblies_Paths $overlayDir $filename 2>/dev/null local exitCode=$? if [ $exitCode == -2146230517 ]; then - echo $filename is not a managed assembly. + echo "$filename is not a managed assembly." elif [ $exitCode != 0 ]; then - echo Unable to precompile $filename. + echo "Unable to precompile $filename." else - echo Successfully precompiled $filename + echo "Successfully precompiled $filename" fi done else - echo Skipping crossgen of FX assemblies. + echo "Skipping crossgen of FX assemblies." fi } @@ -279,8 +214,14 @@ coreClrBinDir= mscorlibDir= coreFxBinDir= uploadToBenchview= -benchViewOS= -runType= +benchViewOS=`lsb_release -i -s``lsb_release -r -s` +runType=local +BENCHVIEW_TOOLS_PATH= +benchViewGroup=CoreCLR +perfCollection= +collectionflags=stopwatch +hasWarmupRun=--drop-first-value +stabilityPrefix= for i in "$@" do @@ -307,12 +248,21 @@ do --coreFxBinDir=*) coreFxBinDir=${i#*=} ;; - --benchViewOS=*) + --benchViewOS=*) benchViewOS=${i#*=} ;; - --runType=*) + --runType=*) runType=${i#*=} ;; + --collectionflags=*) + collectionflags=${i#*=} + ;; + --generatebenchviewdata=*) + BENCHVIEW_TOOLS_PATH=${i#*=} + ;; + --stabilityPrefix=*) + stabilityPrefix=${i#*=} + ;; --uploadToBenchview) uploadToBenchview=TRUE ;; @@ -333,69 +283,97 @@ if [ ! -d "$testRootDir" ]; then echo "Directory specified by --testRootDir does not exist: $testRootDir" exit $EXIT_CODE_EXCEPTION fi - -# Copy native interop test libraries over to the mscorlib path in -# order for interop tests to run on linux. -if [ -z "$mscorlibDir" ]; then - mscorlibDir=$coreClrBinDir +if [ ! -z "$BENCHVIEW_TOOLS_PATH" ] && { [ ! -d "$BENCHVIEW_TOOLS_PATH" ]; }; then + echo BenchView path: "$BENCHVIEW_TOOLS_PATH" was specified, but it does not exist. + exit $EXIT_CODE_EXCEPTION fi -if [ -d "$mscorlibDir" ] && [ -d "$mscorlibDir/bin" ]; then - cp $mscorlibDir/bin/* $mscorlibDir +if [ "$collectionflags" == "stopwatch" ]; then + perfCollection=Off +else + perfCollection=On fi # Install xunit performance packages -export NUGET_PACKAGES=$testNativeBinDir/../../../../packages -echo "NUGET_PACKAGES = $NUGET_PACKAGES" +CORECLR_REPO=$testNativeBinDir/../../../.. +DOTNETCLI_PATH=$CORECLR_REPO/Tools/dotnetcli -pushd $testNativeBinDir/../../../../tests/scripts -$testNativeBinDir/../../../../Tools/dotnetcli/dotnet restore --fallbacksource https://dotnet.myget.org/F/dotnet-buildtools/ --fallbacksource https://dotnet.myget.org/F/dotnet-core/ -popd +export NUGET_PACKAGES=$CORECLR_REPO/packages # Creat coreoverlay dir which contains all dependent binaries -create_core_overlay -precompile_overlay_assemblies -copy_test_native_bin_to_test_root +create_core_overlay || { echo "Creating core overlay failed."; exit 1; } +precompile_overlay_assemblies || { echo "Precompiling overlay assemblies failed."; exit 1; } # Deploy xunit performance packages cd $CORE_ROOT -echo "CORE_ROOT dir = $CORE_ROOT" DO_SETUP=TRUE - if [ ${DO_SETUP} == "TRUE" ]; then -cp $testNativeBinDir/../../../../../packages/Microsoft.DotNet.xunit.performance.runner.cli/1.0.0-alpha-build0040/lib/netstandard1.3/Microsoft.DotNet.xunit.performance.runner.cli.dll . -cp $testNativeBinDir/../../../../../packages/Microsoft.DotNet.xunit.performance.analysis.cli/1.0.0-alpha-build0040/lib/netstandard1.3/Microsoft.DotNet.xunit.performance.analysis.cli.dll . -cp $testNativeBinDir/../../../../../packages/Microsoft.DotNet.xunit.performance.run.core/1.0.0-alpha-build0040/lib/dotnet/*.dll . + $DOTNETCLI_PATH/dotnet restore $CORECLR_REPO/tests/src/Common/PerfHarness/PerfHarness.csproj || { echo "dotnet restore failed."; exit 1; } + $DOTNETCLI_PATH/dotnet publish $CORECLR_REPO/tests/src/Common/PerfHarness/PerfHarness.csproj -c Release -o "$coreOverlayDir" || { echo "dotnet publish failed."; exit 1; } fi # Run coreclr performance tests -echo "Test root dir is: $testRootDir" +echo "Test root dir: $testRootDir" tests=($(find $testRootDir/JIT/Performance/CodeQuality -name '*.exe') $(find $testRootDir/performance/perflab/PerfLab -name '*.dll')) -echo "current dir is $PWD" -rm measurement.json +if [ -f measurement.json ]; then + rm measurement.json || exit $EXIT_CODE_EXCEPTION; +fi + for testcase in ${tests[@]}; do + directory=$(dirname "$testcase") + filename=$(basename "$testcase") + filename="${filename%.*}" -test=$(basename $testcase) -testname=$(basename $testcase .exe) -echo "....Running $testname" -cp $testcase . -cp $testcase-*.txt . - -chmod u+x ./corerun -echo "./corerun Microsoft.DotNet.xunit.performance.runner.cli.dll $test -runner xunit.console.netcore.exe -runnerhost ./corerun -verbose -runid perf-$testname" -./corerun Microsoft.DotNet.xunit.performance.runner.cli.dll $test -runner xunit.console.netcore.exe -runnerhost ./corerun -verbose -runid perf-$testname -echo "./corerun Microsoft.DotNet.xunit.performance.analysis.cli.dll perf-$testname.xml -xml perf-$testname-summary.xml" -./corerun Microsoft.DotNet.xunit.performance.analysis.cli.dll perf-$testname.xml -xml perf-$testname-summary.xml -if [ "$uploadToBenchview" == "TRUE" ] - then - python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/measurement.py xunit perf-$testname.xml --better desc --drop-first-value --append -fi + test=$(basename $testcase) + testname=$(basename $testcase .exe) + + cp $testcase . || exit 1 + if [ stat -t "$directory/$filename"*.txt 1>/dev/null 2>&1 ]; then + cp "$directory/$filename"*.txt . || exit 1 + fi + + # TODO: Do we need this here. + chmod u+x ./corerun + + echo "" + echo "----------" + echo " Running $testname" + echo "----------" + run_command $stabilityPrefix ./corerun PerfHarness.dll $test --perf:runid Perf --perf:collect $collectionflags || exit 1 + if [ -d "$BENCHVIEW_TOOLS_PATH" ]; then + run_command python3.5 "$BENCHVIEW_TOOLS_PATH/measurement.py" xunit "Perf-$filename.xml" --better desc $hasWarmupRun --append || { + echo [ERROR] Failed to generate BenchView data; + exit 1; + } + fi + + # Rename file to be archived by Jenkins. + mv -f "Perf-$filename.xml" "$CORECLR_REPO/Perf-$filename-$perfCollection.xml" || { + echo [ERROR] Failed to move "Perf-$filename.xml" to "$CORECLR_REPO". + exit 1; + } done -if [ "$uploadToBenchview" == "TRUE" ] - then - python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission.py measurement.json --build ../../../../../build.json --machine-data ../../../../../machinedata.json --metadata ../../../../../submission-metadata.json --group "CoreCLR" --type "$runType" --config-name "Release" --config Configuration "Release" --config OS "$benchViewOS" --arch "x64" --machinepool "Perfsnake" - python3.5 ../../../../../tests/scripts/Microsoft.BenchView.JSONFormat/tools/upload.py submission.json --container coreclr + +if [ -d "$BENCHVIEW_TOOLS_PATH" ]; then + args=measurement.json + args+=" --build ../../../../../build.json" + args+=" --machine-data ../../../../../machinedata.json" + args+=" --metadata ../../../../../submission-metadata.json" + args+=" --group $benchViewGroup" + args+=" --type $runType" + args+=" --config-name Release" + args+=" --config Configuration Release" + args+=" --config OS $benchViewOS" + args+=" --config Profile $perfCollection" + args+=" --arch x64" + args+=" --machinepool Perfsnake" + run_command python3.5 "$BENCHVIEW_TOOLS_PATH/submission.py" $args || { + echo [ERROR] Failed to generate BenchView submission data; + exit 1; + } +fi + +if [ -d "$BENCHVIEW_TOOLS_PATH" ] && { [ "$uploadToBenchview" == "TRUE" ]; }; then + run_command python3.5 "$BENCHVIEW_TOOLS_PATH/upload.py" submission.json --container coreclr fi -mkdir ../../../../../sandbox -cp *.xml ../../../../../sandbox diff --git a/tests/src/Common/PerfHarness/PerfHarness.csproj b/tests/src/Common/PerfHarness/PerfHarness.csproj index 33b55a86a297..a15e8022aea2 100644 --- a/tests/src/Common/PerfHarness/PerfHarness.csproj +++ b/tests/src/Common/PerfHarness/PerfHarness.csproj @@ -7,7 +7,7 @@ - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 diff --git a/tests/src/Common/external/external.depproj b/tests/src/Common/external/external.depproj index 797046f2a99b..464d27e01948 100644 --- a/tests/src/Common/external/external.depproj +++ b/tests/src/Common/external/external.depproj @@ -22,16 +22,16 @@ 1.1.1 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 1.0.3-alpha-experimental diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Adams/Adams.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Adams/Adams.cs index 431d857f16c1..3f21d9cd4c9d 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Adams/Adams.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Adams/Adams.cs @@ -15,7 +15,6 @@ #if XUNIT_PERF [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] #endif // XUNIT_PERF namespace Benchstone.BenchF diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMk2/BenchMk2.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMk2/BenchMk2.cs index c4cfc0b217aa..5c616a5b6c37 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMk2/BenchMk2.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMk2/BenchMk2.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMrk/BenchMrk.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMrk/BenchMrk.cs index f342b2d84ebc..ec70102f27f9 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMrk/BenchMrk.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/BenchMrk/BenchMrk.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Bisect/Bisect.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Bisect/Bisect.cs index 724416a6265b..cd44b1c83eba 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Bisect/Bisect.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Bisect/Bisect.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/DMath/DMath.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/DMath/DMath.cs index 47c5e2028531..dbce0240992e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/DMath/DMath.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/DMath/DMath.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/FFT/FFT.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/FFT/FFT.cs index 8881eae6c682..3ddd75491d86 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/FFT/FFT.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/FFT/FFT.cs @@ -11,7 +11,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/InProd/InProd.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/InProd/InProd.cs index 26096b736e1a..08665a8950ae 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/InProd/InProd.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/InProd/InProd.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/InvMt/InvMt.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/InvMt/InvMt.cs index 6ecfb62d9168..ee0af77b59c4 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/InvMt/InvMt.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/InvMt/InvMt.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/LLoops/LLoops.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/LLoops/LLoops.cs index 73ec09aa04a4..d7d7eb82596e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/LLoops/LLoops.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/LLoops/LLoops.cs @@ -58,7 +58,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Lorenz/Lorenz.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Lorenz/Lorenz.cs index 5819446bf05d..5fe7ef34e4cf 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Lorenz/Lorenz.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Lorenz/Lorenz.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/MatInv4/MatInv4.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/MatInv4/MatInv4.cs index a00c29b843d3..02588e18edfa 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/MatInv4/MatInv4.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/MatInv4/MatInv4.cs @@ -8,7 +8,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/NewtE/NewtE.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/NewtE/NewtE.cs index 47f90d86160f..cc698f44dd7a 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/NewtE/NewtE.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/NewtE/NewtE.cs @@ -11,7 +11,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/NewtR/NewtR.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/NewtR/NewtR.cs index c590f6e570ec..bfb6546a388d 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/NewtR/NewtR.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/NewtR/NewtR.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Regula/Regula.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Regula/Regula.cs index 0011288e552e..c2978914b281 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Regula/Regula.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Regula/Regula.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Romber/Romber.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Romber/Romber.cs index a4d56f7b0fdd..1e7c6879ad9e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Romber/Romber.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Romber/Romber.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Secant/Secant.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Secant/Secant.cs index 060fb5b0a4d4..9dcc1feb3fab 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Secant/Secant.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Secant/Secant.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Simpsn/Simpsn.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Simpsn/Simpsn.cs index 5b2df292af64..34c6c56230d1 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Simpsn/Simpsn.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Simpsn/Simpsn.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/SqMtx/SqMtx.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/SqMtx/SqMtx.cs index b22aad88b7a2..8ea7556d5603 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/SqMtx/SqMtx.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/SqMtx/SqMtx.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Trap/Trap.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Trap/Trap.cs index 3a0de102edd2..33895ccc0cb5 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Trap/Trap.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Trap/Trap.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchF/Whetsto/Whetsto.cs b/tests/src/JIT/Performance/CodeQuality/BenchF/Whetsto/Whetsto.cs index ffe3fa5d8b50..d72e58789db5 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchF/Whetsto/Whetsto.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchF/Whetsto/Whetsto.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchF { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/8Queens/8Queens.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/8Queens/8Queens.cs index e641ee0a7445..d4994418228b 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/8Queens/8Queens.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/8Queens/8Queens.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Ackermann/Ackermann.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Ackermann/Ackermann.cs index 5bfa19652740..5188571c91f0 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Ackermann/Ackermann.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Ackermann/Ackermann.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray/AddArray.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray/AddArray.cs index d38e6b44759e..16da08250d88 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray/AddArray.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray/AddArray.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray2/AddArray2.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray2/AddArray2.cs index a6004f030945..3ce1a98f9fb2 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray2/AddArray2.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/AddArray2/AddArray2.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Array1/Array1.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Array1/Array1.cs index 780b7ce66ff0..5e383ace66bb 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Array1/Array1.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Array1/Array1.cs @@ -16,7 +16,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Array2/Array2.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Array2/Array2.cs index dac100c5a70b..45a4a9765b3b 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Array2/Array2.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Array2/Array2.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/BenchE/BenchE.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/BenchE/BenchE.cs index b16c48280bce..5d9a7f10cb64 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/BenchE/BenchE.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/BenchE/BenchE.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort/BubbleSort.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort/BubbleSort.cs index 558484d3d6f1..a2fbbcee3077 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort/BubbleSort.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort/BubbleSort.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort2/BubbleSort2.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort2/BubbleSort2.cs index 8c2e63a95414..2dba59a793ce 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort2/BubbleSort2.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/BubbleSort2/BubbleSort2.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/CSieve/CSieve.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/CSieve/CSieve.cs index aca263711ddf..67544bae599f 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/CSieve/CSieve.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/CSieve/CSieve.cs @@ -10,7 +10,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Fib/Fib.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Fib/Fib.cs index ee028db20f99..3dd897db0f68 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Fib/Fib.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Fib/Fib.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/HeapSort/HeapSort.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/HeapSort/HeapSort.cs index 64b5c8f33b00..b33087bd8e06 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/HeapSort/HeapSort.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/HeapSort/HeapSort.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/IniArray/IniArray.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/IniArray/IniArray.cs index 268af7b77987..ff878d612720 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/IniArray/IniArray.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/IniArray/IniArray.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/LogicArray/LogicArray.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/LogicArray/LogicArray.cs index c5b4e3b0d8be..c82b1e1e8b1a 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/LogicArray/LogicArray.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/LogicArray/LogicArray.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Midpoint/Midpoint.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Midpoint/Midpoint.cs index 9146874358ef..f04fa9ff6036 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Midpoint/Midpoint.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Midpoint/Midpoint.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/MulMatrix/MulMatrix.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/MulMatrix/MulMatrix.cs index a45c01298445..202ed3c05225 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/MulMatrix/MulMatrix.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/MulMatrix/MulMatrix.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/NDhrystone/NDhrystone.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/NDhrystone/NDhrystone.cs index 33effcbf4013..7a28c97cf72d 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/NDhrystone/NDhrystone.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/NDhrystone/NDhrystone.cs @@ -14,7 +14,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Permutate/Permutate.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Permutate/Permutate.cs index 43fc28b37067..5dda2631b227 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Permutate/Permutate.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Permutate/Permutate.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Pi/Pi.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Pi/Pi.cs index 2fe0918784d2..7a6bed7dd74b 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Pi/Pi.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Pi/Pi.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/Puzzle/Puzzle.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/Puzzle/Puzzle.cs index 2271e0245539..220b5c70234b 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/Puzzle/Puzzle.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/Puzzle/Puzzle.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/QuickSort/QuickSort.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/QuickSort/QuickSort.cs index 7cdac96a6925..b5bbc479131a 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/QuickSort/QuickSort.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/QuickSort/QuickSort.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/TreeInsert/TreeInsert.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/TreeInsert/TreeInsert.cs index 2867abc4d415..76d04d607a05 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/TreeInsert/TreeInsert.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/TreeInsert/TreeInsert.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/TreeSort/TreeSort.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/TreeSort/TreeSort.cs index e0971346cf8d..07c7c298772e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/TreeSort/TreeSort.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/TreeSort/TreeSort.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchI/XposMatrix/XposMatrix.cs b/tests/src/JIT/Performance/CodeQuality/BenchI/XposMatrix/XposMatrix.cs index de8b4860497f..47fbf3730a29 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchI/XposMatrix/XposMatrix.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchI/XposMatrix/XposMatrix.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Benchstone.BenchI { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp.cs index 05dbe94f093a..6e446294995f 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp.cs @@ -15,7 +15,6 @@ contributed by Marek Safar using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] [assembly: MeasureGCCounts] namespace BenchmarksGame diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp3.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp3.cs index 6dcf48a42215..0cc239274be4 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp3.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csharp3.cs @@ -16,7 +16,6 @@ and optimized by kasthack using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] [assembly: MeasureGCCounts] namespace BenchmarksGame diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fasta/fasta.csharp-2.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fasta/fasta.csharp-2.cs index 92fbceb6921f..3e97d6442915 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fasta/fasta.csharp-2.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fasta/fasta.csharp-2.cs @@ -16,7 +16,6 @@ optimizations by Alp Toker using System.Text; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fastaredux/fastaredux.csharp.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fastaredux/fastaredux.csharp.cs index df8999a2deaa..c7a786a77ef3 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fastaredux/fastaredux.csharp.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/fastaredux/fastaredux.csharp.cs @@ -16,7 +16,6 @@ contributed by Robert F. Tobler using System.Text; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/k-nucleotide.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/k-nucleotide.cs index 54bd10702cff..3325a619265e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/k-nucleotide.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/k-nucleotide.cs @@ -2,7 +2,7 @@ http://benchmarksgame.alioth.debian.org/ * * submitted by Josh Goldfoot - * + * */ using System; @@ -14,7 +14,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] [assembly: MeasureGCCounts] namespace BenchmarksGame @@ -24,13 +23,13 @@ public class knucleotide { #if DEBUG const int Iterations = 1; - const string InputFile = "knucleotide-input.txt"; + const string InputFile = "knucleotide-input.txt"; static int[] expectedCountLetter = new int[] { 1480, 974, 970, 1576 }; static int[] expectedCountPairs = new int[] { 420, 272, 292, 496, 273, 202, 201, 298, 316, 185, 167, 302, 470, 315, 310, 480 }; static int[] expectedCountFragments = new int[] { 54, 24, 4, 0, 0 }; #else const int Iterations = 10; - const string InputFile = "knucleotide-input-big.txt"; + const string InputFile = "knucleotide-input-big.txt"; static int[] expectedCountLetter = new int[] { 302923, 198136, 197566, 301375 }; static int[] expectedCountPairs = new int[] { 91779, 60030, 59889, 91225, 60096, 39203, 39081, 59756, 59795, 39190, 39023, 59557, 91253, 59713, 59572, 90837 }; static int[] expectedCountFragments = new int[] { 11765, 3572, 380, 7, 7 }; @@ -90,7 +89,7 @@ static string FindInput(string s) public static int Main(string[] args) { int iterations = Iterations; - + string inputFile = FindInput(InputFile); if (inputFile == null) { @@ -258,7 +257,7 @@ public static void Bench_Parallel() { throw new Exception("unable to find input"); } - foreach (var iteration in Benchmark.Iterations) + foreach (var iteration in Benchmark.Iterations) { using (iteration.StartMeasurement()) { @@ -284,7 +283,7 @@ public static void Bench_No_Parallel() { throw new Exception("unable to find input"); } - foreach (var iteration in Benchmark.Iterations) + foreach (var iteration in Benchmark.Iterations) { using (iteration.StartMeasurement()) { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/nbody/nbody.csharp-3.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/nbody/nbody.csharp-3.cs index 2124cf4d9efd..60e083ad11ba 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/nbody/nbody.csharp-3.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/nbody/nbody.csharp-3.cs @@ -13,7 +13,6 @@ using System; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/pidigits/pi-digits.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/pidigits/pi-digits.cs index 7e36f0e42438..3c79a89edc8e 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/pidigits/pi-digits.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/pidigits/pi-digits.cs @@ -18,7 +18,6 @@ using System.Text; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna.csharp-6.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna.csharp-6.cs index 19928012ba68..882bfd23f975 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna.csharp-6.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna.csharp-6.cs @@ -19,7 +19,6 @@ using System.Threading; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp.csharp-1.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp.csharp-1.cs index 4e875742e8c6..bca63bd00e21 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp.csharp-1.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp.csharp-1.cs @@ -16,7 +16,6 @@ contributed by Robert F. Tobler to process large blocks of byte arrays using System.IO; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/spectralnorm/spectralnorm.cs b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/spectralnorm/spectralnorm.cs index 01eeea07bfc7..5f2b1aed5f67 100644 --- a/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/spectralnorm/spectralnorm.cs +++ b/tests/src/JIT/Performance/CodeQuality/BenchmarksGame/spectralnorm/spectralnorm.cs @@ -13,7 +13,6 @@ contributed by Isaac Gouy using System; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace BenchmarksGame { diff --git a/tests/src/JIT/Performance/CodeQuality/Burgers/Burgers.cs b/tests/src/JIT/Performance/CodeQuality/Burgers/Burgers.cs index 02dafa2dda50..9880ee511fcc 100644 --- a/tests/src/JIT/Performance/CodeQuality/Burgers/Burgers.cs +++ b/tests/src/JIT/Performance/CodeQuality/Burgers/Burgers.cs @@ -4,7 +4,7 @@ // // .NET SIMD to solve Burgers' equation // -// Benchmark based on +// Benchmark based on // https://github.com/taumuon/SIMD-Vectorisation-Burgers-Equation-CSharp // http://www.taumuon.co.uk/2014/10/net-simd-to-solve-burgers-equation.html @@ -15,7 +15,6 @@ using System.Runtime.CompilerServices; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] public class Burgers { @@ -192,7 +191,7 @@ public static int Main() double[] x = linspace(0.0, 2.0 * Math.PI, nx); double[] initial = GetAnalytical(x, 0.0, nu); - // Warmup + // Warmup GetCalculated0(1, nx, dx, dt, nu, initial); GetCalculated1(1, nx, dx, dt, nu, initial); diff --git a/tests/src/JIT/Performance/CodeQuality/Bytemark/ByteMark.cs b/tests/src/JIT/Performance/CodeQuality/Bytemark/ByteMark.cs index 280744bf5b61..79e7c5e7095e 100644 --- a/tests/src/JIT/Performance/CodeQuality/Bytemark/ByteMark.cs +++ b/tests/src/JIT/Performance/CodeQuality/Bytemark/ByteMark.cs @@ -3,7 +3,7 @@ // See the LICENSE file in the project root for more information. /* ** This program was translated to C# and adapted for xunit-performance. -** New variants of several tests were added to compare class versus +** New variants of several tests were added to compare class versus ** struct and to compare jagged arrays vs multi-dimensional arrays. */ @@ -24,7 +24,7 @@ ** are error-free. Consequently, McGraw-HIll and BYTE Magazine make ** no claims in regard to the fitness of the source code, executable ** code, and documentation of the BYTEmark. -** +** ** Furthermore, BYTE Magazine, McGraw-Hill, and all employees ** of McGraw-Hill cannot be held responsible for any damages resulting ** from the use of this code or the results obtained from using @@ -36,7 +36,6 @@ using System.IO; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] internal class global { @@ -1273,7 +1272,7 @@ public static void BenchNumericSortJagged() { using (iteration.StartMeasurement()) { - for (int i = 0; i < NumericSortJaggedIterations; i++) + for (int i = 0; i < NumericSortJaggedIterations; i++) { t.Run(); } @@ -1295,7 +1294,7 @@ public static void BenchNumericSortRectangular() { using (iteration.StartMeasurement()) { - for (int i = 0; i < NumericSortRectangularIterations; i++) + for (int i = 0; i < NumericSortRectangularIterations; i++) { t.Run(); } @@ -1317,7 +1316,7 @@ public static void BenchStringSort() { using (iteration.StartMeasurement()) { - for (int i = 0; i < StringSortIterations; i++) + for (int i = 0; i < StringSortIterations; i++) { t.Run(); } @@ -1338,7 +1337,7 @@ public static void BenchBitOps() { using (iteration.StartMeasurement()) { - for (int i = 0; i < BitOpsIterations; i++) + for (int i = 0; i < BitOpsIterations; i++) { t.Run(); } @@ -1360,7 +1359,7 @@ public static void BenchEmFloat() { using (iteration.StartMeasurement()) { - for (int i = 0; i < EmFloatIterations; i++) + for (int i = 0; i < EmFloatIterations; i++) { t.Run(); } @@ -1382,7 +1381,7 @@ public static void BenchEmFloatClass() { using (iteration.StartMeasurement()) { - for (int i = 0; i < EmFloatClassIterations; i++) + for (int i = 0; i < EmFloatClassIterations; i++) { t.Run(); } @@ -1403,7 +1402,7 @@ public static void BenchFourier() { using (iteration.StartMeasurement()) { - for (int i = 0; i < FourierIterations; i++) + for (int i = 0; i < FourierIterations; i++) { t.Run(); } @@ -1425,7 +1424,7 @@ public static void BenchAssignJagged() { using (iteration.StartMeasurement()) { - for (int i = 0; i < AssignJaggedIterations; i++) + for (int i = 0; i < AssignJaggedIterations; i++) { t.Run(); } @@ -1447,7 +1446,7 @@ public static void BenchAssignRectangular() { using (iteration.StartMeasurement()) { - for (int i = 0; i < AssignRectangularIterations; i++) + for (int i = 0; i < AssignRectangularIterations; i++) { t.Run(); } @@ -1469,7 +1468,7 @@ public static void BenchIDEAEncryption() { using (iteration.StartMeasurement()) { - for (int i = 0; i < IDEAEncryptionIterations; i++) + for (int i = 0; i < IDEAEncryptionIterations; i++) { t.Run(); } @@ -1491,7 +1490,7 @@ public static void BenchNeuralJagged() { using (iteration.StartMeasurement()) { - for (int i = 0; i < NeuralJaggedIterations; i++) + for (int i = 0; i < NeuralJaggedIterations; i++) { t.Run(); } @@ -1513,7 +1512,7 @@ public static void BenchNeural() { using (iteration.StartMeasurement()) { - for (int i = 0; i < NeuralIterations; i++) + for (int i = 0; i < NeuralIterations; i++) { t.Run(); } @@ -1535,7 +1534,7 @@ public static void BenchLUDecomp() { using (iteration.StartMeasurement()) { - for (int i = 0; i < LUDecompIterations; i++) + for (int i = 0; i < LUDecompIterations; i++) { t.Run(); } diff --git a/tests/src/JIT/Performance/CodeQuality/FractalPerf/FractalPerf.cs b/tests/src/JIT/Performance/CodeQuality/FractalPerf/FractalPerf.cs index cbaad45df511..d5761da4c2aa 100644 --- a/tests/src/JIT/Performance/CodeQuality/FractalPerf/FractalPerf.cs +++ b/tests/src/JIT/Performance/CodeQuality/FractalPerf/FractalPerf.cs @@ -13,7 +13,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace FractalPerf { @@ -103,11 +102,11 @@ public override double Render() { // set the Julia Set constant complex seed = new complex(Real, Imaginary); - // run through every point on the screen, setting + // run through every point on the screen, setting // m and n to the coordinates for (double m = XB; m < XE; m += XS) { for (double n = YB; n < YE; n += YS) { - // the initial z value is the current pixel, + // the initial z value is the current pixel, // so x and y have to be set to m and n complex accum = new complex(m, n); // perform the iteration @@ -120,8 +119,8 @@ public override double Render() { accum = accum.square() + seed; } // determine the color using the number of - // iterations it took for the number to become too big - // char color = num % number_of_colors; + // iterations it took for the number to become too big + // char color = num % number_of_colors; // plot the point result += num; } @@ -168,7 +167,7 @@ static bool TestBase() { } return result; } - + public static int Main() { bool result = TestBase(); return (result ? 100 : -1); diff --git a/tests/src/JIT/Performance/CodeQuality/Inlining/InlineGCStruct.cs b/tests/src/JIT/Performance/CodeQuality/Inlining/InlineGCStruct.cs index 660627cc7222..1439c77dd9e8 100644 --- a/tests/src/JIT/Performance/CodeQuality/Inlining/InlineGCStruct.cs +++ b/tests/src/JIT/Performance/CodeQuality/Inlining/InlineGCStruct.cs @@ -20,7 +20,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Inlining { diff --git a/tests/src/JIT/Performance/CodeQuality/Inlining/NoThrowInline.cs b/tests/src/JIT/Performance/CodeQuality/Inlining/NoThrowInline.cs index dc05f44b339c..f28d4caf8119 100644 --- a/tests/src/JIT/Performance/CodeQuality/Inlining/NoThrowInline.cs +++ b/tests/src/JIT/Performance/CodeQuality/Inlining/NoThrowInline.cs @@ -11,7 +11,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Inlining { diff --git a/tests/src/JIT/Performance/CodeQuality/Linq/Linq.cs b/tests/src/JIT/Performance/CodeQuality/Linq/Linq.cs index 58ea46a0c982..2a0fd27890ed 100644 --- a/tests/src/JIT/Performance/CodeQuality/Linq/Linq.cs +++ b/tests/src/JIT/Performance/CodeQuality/Linq/Linq.cs @@ -9,7 +9,6 @@ using System.Linq; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] public class Product { diff --git a/tests/src/JIT/Performance/CodeQuality/Math/Functions/Functions.cs b/tests/src/JIT/Performance/CodeQuality/Math/Functions/Functions.cs index c9b1ef114782..aff91ba844ca 100644 --- a/tests/src/JIT/Performance/CodeQuality/Math/Functions/Functions.cs +++ b/tests/src/JIT/Performance/CodeQuality/Math/Functions/Functions.cs @@ -8,7 +8,6 @@ using Microsoft.Xunit.Performance; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Functions { diff --git a/tests/src/JIT/Performance/CodeQuality/Roslyn/CscBench.cs b/tests/src/JIT/Performance/CodeQuality/Roslyn/CscBench.cs index e3337d115b85..704e728c55bd 100644 --- a/tests/src/JIT/Performance/CodeQuality/Roslyn/CscBench.cs +++ b/tests/src/JIT/Performance/CodeQuality/Roslyn/CscBench.cs @@ -13,7 +13,6 @@ using System.Runtime.CompilerServices; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] public static class CscBench { diff --git a/tests/src/JIT/Performance/CodeQuality/SIMD/ConsoleMandel/ConsoleMandel.cs b/tests/src/JIT/Performance/CodeQuality/SIMD/ConsoleMandel/ConsoleMandel.cs index 5ab08202b652..c5058dcfac13 100644 --- a/tests/src/JIT/Performance/CodeQuality/SIMD/ConsoleMandel/ConsoleMandel.cs +++ b/tests/src/JIT/Performance/CodeQuality/SIMD/ConsoleMandel/ConsoleMandel.cs @@ -9,7 +9,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace SIMD { diff --git a/tests/src/JIT/Performance/CodeQuality/SIMD/RayTracer/RayTracerBench.cs b/tests/src/JIT/Performance/CodeQuality/SIMD/RayTracer/RayTracerBench.cs index 254672b46254..715e0e2a89de 100644 --- a/tests/src/JIT/Performance/CodeQuality/SIMD/RayTracer/RayTracerBench.cs +++ b/tests/src/JIT/Performance/CodeQuality/SIMD/RayTracer/RayTracerBench.cs @@ -14,7 +14,6 @@ using System.Collections.Concurrent; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace SIMD { diff --git a/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs b/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs index 81d9c0952f6a..bfa1f43a5ff1 100644 --- a/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs +++ b/tests/src/JIT/Performance/CodeQuality/SIMD/SeekUnroll/SeekUnroll.cs @@ -12,7 +12,6 @@ using Xunit; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] public static class SeekUnroll { diff --git a/tests/src/JIT/Performance/CodeQuality/SciMark/kernel.cs b/tests/src/JIT/Performance/CodeQuality/SciMark/kernel.cs index 3cba0c16c8a4..e06d7ff093ef 100644 --- a/tests/src/JIT/Performance/CodeQuality/SciMark/kernel.cs +++ b/tests/src/JIT/Performance/CodeQuality/SciMark/kernel.cs @@ -4,9 +4,9 @@ /// /// This is a port of the SciMark2a Java Benchmark to C# by /// Chris Re (cmr28@cornell.edu) and Werner Vogels (vogels@cs.cornell.edu) -/// +/// /// For details on the original authors see http://math.nist.gov/scimark2 -/// +/// /// This software is likely to burn your processor, bitflip your memory chips /// anihilate your screen and corrupt all your disks, so you it at your /// own risk. @@ -17,7 +17,6 @@ using System; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace SciMark2 { @@ -217,7 +216,7 @@ public static double measureSparseMatmult(int N, int nz, double min_time, Random // +** * * + // +* * * * + // +* * * * + - // +* * * * + + // +* * * * + // +-----------------+ // // (as best reproducible with integer artihmetic) diff --git a/tests/src/JIT/Performance/CodeQuality/Serialization/Deserialize.cs b/tests/src/JIT/Performance/CodeQuality/Serialization/Deserialize.cs index 56fc2665add7..56581616443e 100644 --- a/tests/src/JIT/Performance/CodeQuality/Serialization/Deserialize.cs +++ b/tests/src/JIT/Performance/CodeQuality/Serialization/Deserialize.cs @@ -13,7 +13,6 @@ using Microsoft.Xunit.Performance; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Serialization { diff --git a/tests/src/JIT/Performance/CodeQuality/Serialization/Serialize.cs b/tests/src/JIT/Performance/CodeQuality/Serialization/Serialize.cs index 39ab5876c9a1..6e2c41a6d1f5 100644 --- a/tests/src/JIT/Performance/CodeQuality/Serialization/Serialize.cs +++ b/tests/src/JIT/Performance/CodeQuality/Serialization/Serialize.cs @@ -12,7 +12,6 @@ using Microsoft.Xunit.Performance; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Serialization { diff --git a/tests/src/JIT/Performance/CodeQuality/Span/Indexer.cs b/tests/src/JIT/Performance/CodeQuality/Span/Indexer.cs index 4a7264ba925f..c7f4846548b3 100644 --- a/tests/src/JIT/Performance/CodeQuality/Span/Indexer.cs +++ b/tests/src/JIT/Performance/CodeQuality/Span/Indexer.cs @@ -14,7 +14,6 @@ using Microsoft.Xunit.Performance; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Span { @@ -526,7 +525,7 @@ public static void KnownSizeArray(int length) Invoke((int innerIterationCount) => { byte result = TestKnownSizeArray(innerIterationCount); - return result; + return result; }, "KnownSizeArray({0})", length); } @@ -1014,7 +1013,7 @@ public static int Main(string[] args) Console.WriteLine("Some tests failed validation"); return -1; } - + return 100; } } diff --git a/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs b/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs index d84152132db7..ef694421ef3e 100644 --- a/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs +++ b/tests/src/JIT/Performance/CodeQuality/Span/SpanBench.cs @@ -14,7 +14,6 @@ using Microsoft.Xunit.Performance; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace Span { @@ -1109,7 +1108,7 @@ static void TestSpanNonPortableCast(From[] array, int iterationCount, #endregion #region TestSpanAsSpanStringChar - + [Benchmark(InnerIterationCount = BaseIterations)] [InlineData(1)] [InlineData(10)] @@ -1146,7 +1145,7 @@ static void TestSpanAsSpanStringChar(string s, int iterationCount, bool untrue) } } - #endregion + #endregion #endregion // TestSpanAPIs @@ -1186,7 +1185,7 @@ public static int Main(string[] args) } } } - + // The only failure modes are crash/exception. return 100; } diff --git a/tests/src/JIT/Performance/CodeQuality/V8/Crypto/Crypto.cs b/tests/src/JIT/Performance/CodeQuality/V8/Crypto/Crypto.cs index 620e01284846..f49f92f8db6f 100644 --- a/tests/src/JIT/Performance/CodeQuality/V8/Crypto/Crypto.cs +++ b/tests/src/JIT/Performance/CodeQuality/V8/Crypto/Crypto.cs @@ -44,7 +44,6 @@ using System.Globalization; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] namespace V8.Crypto { diff --git a/tests/src/JIT/Performance/CodeQuality/V8/Richards/Richards.cs b/tests/src/JIT/Performance/CodeQuality/V8/Richards/Richards.cs index 5095d22ad749..33f00ad8aa94 100644 --- a/tests/src/JIT/Performance/CodeQuality/V8/Richards/Richards.cs +++ b/tests/src/JIT/Performance/CodeQuality/V8/Richards/Richards.cs @@ -15,7 +15,6 @@ using System.Collections.Generic; [assembly: OptimizeForBenchmarks] -[assembly: MeasureInstructionsRetired] // using System.Diagnostics; // using System.Text.RegularExpressions; diff --git a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj index e9ac1d58e7cb..27af3866332f 100644 --- a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj +++ b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj @@ -11,16 +11,16 @@ 1.1.1 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 1.0.3-alpha-experimental diff --git a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj index c57bef951049..20accfe300a8 100644 --- a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj +++ b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj @@ -8,16 +8,16 @@ - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 1.0.3-alpha-experimental diff --git a/tests/src/performance/performance.csproj b/tests/src/performance/performance.csproj index c1ca34cd343c..5a6ca1279fac 100644 --- a/tests/src/performance/performance.csproj +++ b/tests/src/performance/performance.csproj @@ -8,16 +8,16 @@ - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 - 1.0.0-beta-build0004 + 1.0.0-beta-build0006 1.0.3-alpha-experimental From cdc0e40d477c8f4a677314085677e760ae384f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Rivero?= Date: Thu, 1 Jun 2017 08:56:27 -0700 Subject: [PATCH 2/2] Updating version of xUnit-Performance-Api (#11987) - This update contains a fix for this issue: https://github.com/Microsoft/xunit-performance/issues/230 - Consolidate the Api version in two places: `dependencies.prop` and `PerfHarness.csproj` - Move performance log files to root directory where Jenkins can archive them - Fixed the output to console, so it is easier to see on the Jenkins job the sccript steps live and get an exact repro step if needed --- dependencies.props | 13 +++- perf.groovy | 6 +- tests/scripts/run-xunit-perf.cmd | 74 +++++++++---------- tests/scripts/run-xunit-perf.sh | 6 +- .../src/Common/PerfHarness/PerfHarness.csproj | 4 +- tests/src/Common/external/external.depproj | 12 +-- .../benchmark+roslyn/benchmark+roslyn.csproj | 12 +-- .../benchmark+serialize.csproj | 12 +-- .../src/JIT/config/benchmark/benchmark.csproj | 14 ++-- tests/src/performance/performance.csproj | 12 +-- 10 files changed, 90 insertions(+), 75 deletions(-) diff --git a/dependencies.props b/dependencies.props index 528b6cc72e2e..abc0a85f6823 100644 --- a/dependencies.props +++ b/dependencies.props @@ -14,6 +14,9 @@ 2.0.0-preview2-25330-02 2.2.0-beta2-build3300 + 1.0.2-prerelease-00177 + 1.0.0-beta-build0007 + 1.0.3-alpha-experimental @@ -72,8 +75,16 @@ 1.0.0-alpha-build0040 + + + + + + $(XUnitPerformanceApiVersion) + + - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) diff --git a/perf.groovy b/perf.groovy index 125ed7b06218..9be47c0b3cac 100644 --- a/perf.groovy +++ b/perf.groovy @@ -105,12 +105,12 @@ def static getOSGroup(def os) { batchFile("tests\\runtest.cmd ${configuration} ${architecture} GenerateLayoutOnly") - // Run with just stopwatch + // Run with just stopwatch: Profile=Off batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\linkbench\\linkbench -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -nowarmup -runtype ${runType} -scenarioTest -group ILLink") - // Run with the full set of counters enabled + // Run with the full set of counters enabled: Profile=On batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\performance\\perflab\\Perflab -library -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") batchFile("tests\\scripts\\run-xunit-perf.cmd -arch ${arch} -configuration ${configuration} -testBinLoc bin\\tests\\${os}.${architecture}.${configuration}\\Jit\\Performance\\CodeQuality -generateBenchviewData \"%WORKSPACE%\\Microsoft.Benchview.JSONFormat\\tools\" ${uploadString} -runtype ${runType} -collectionFlags default+BranchMispredictions+CacheMisses+InstructionRetired+gcapi -stabilityPrefix \"START \"CORECLR_PERF_RUN\" /B /WAIT /HIGH /AFFINITY 0x2\"") } @@ -124,6 +124,7 @@ def static getOSGroup(def os) { def archiveSettings = new ArchivalSettings() archiveSettings.addFiles('Perf-*.xml') archiveSettings.addFiles('Perf-*.etl') + archiveSettings.addFiles('Perf-*.log') archiveSettings.addFiles('machinedata.json') Utilities.addArchival(newJob, archiveSettings) @@ -288,6 +289,7 @@ def static getOSGroup(def os) { // Save machinedata.json to /artifact/bin/ Jenkins dir def archiveSettings = new ArchivalSettings() + archiveSettings.addFiles('Perf-*.log') archiveSettings.addFiles('Perf-*.xml') archiveSettings.addFiles('machinedata.json') Utilities.addArchival(newJob, archiveSettings) diff --git a/tests/scripts/run-xunit-perf.cmd b/tests/scripts/run-xunit-perf.cmd index 7b1bd3e9b001..e223a3bda94e 100644 --- a/tests/scripts/run-xunit-perf.cmd +++ b/tests/scripts/run-xunit-perf.cmd @@ -6,7 +6,6 @@ @if defined _echo echo on setlocal ENABLEDELAYEDEXPANSION -setlocal set ERRORLEVEL= set BENCHVIEW_RUN_TYPE=local set CORECLR_REPO=%CD% @@ -45,7 +44,6 @@ setlocal rem find and stage the tests set /A "LV_FAILURES=0" for /R %CORECLR_PERF% %%T in (*.%TEST_FILE_EXT%) do ( - rem Skip known failures call :run_benchmark %%T || ( set /A "LV_FAILURES+=1" ) @@ -73,13 +71,11 @@ setlocal set BENCHDIR=%~p1 rem copy benchmark and any input files - call :run_cmd xcopy /s %~1 . >> %RUNLOG% || exit /b 1 + call :run_cmd xcopy /sy %~1 . >> %RUNLOG% || exit /b 1 if exist "%BENCHDIR%*.txt" ( - call :run_cmd xcopy /s %BENCHDIR%*.txt . >> %RUNLOG% || exit /b 1 + call :run_cmd xcopy /sy %BENCHDIR%*.txt . >> %RUNLOG% || exit /b 1 ) - set CORE_ROOT=%CORECLR_REPO%\sandbox - rem setup additional environment variables if DEFINED TEST_ENV ( if EXIST "%TEST_ENV%" ( @@ -87,13 +83,26 @@ setlocal ) ) - set BENCHNAME_LOG_FILE_NAME=%BENCHNAME%.log + echo/ + echo/ ---------- + echo/ Running %BENCHNAME% + echo/ ---------- + + rem CORE_ROOT environment variable is used by some benchmarks such as Roslyn / CscBench. + set CORE_ROOT=%CORECLR_REPO%\sandbox + + set LV_RUNID=Perf-%ETW_COLLECTION% + set BENCHNAME_LOG_FILE_NAME=%LV_RUNID%-%BENCHNAME%.log + set LV_CMD= if defined IS_SCENARIO_TEST ( - call :run_cmd corerun.exe "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 + set "LV_CMD=corerun.exe "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid "%LV_RUNID%"" ) else ( - call :run_cmd %STABILITY_PREFIX% corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid Perf --perf:collect %COLLECTION_FLAGS% 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 + set "LV_CMD=%STABILITY_PREFIX% corerun.exe PerfHarness.dll "%CORECLR_REPO%\sandbox\%BENCHNAME%.%TEST_FILE_EXT%" --perf:runid "%LV_RUNID%" --perf:collect %COLLECTION_FLAGS%" ) + call :print_to_console $ !LV_CMD! + call :run_cmd !LV_CMD! 1>"%BENCHNAME_LOG_FILE_NAME%" 2>&1 + IF %ERRORLEVEL% NEQ 0 ( call :print_error corerun.exe exited with %ERRORLEVEL% code. if exist "%BENCHNAME_LOG_FILE_NAME%" type "%BENCHNAME_LOG_FILE_NAME%" @@ -103,14 +112,13 @@ setlocal rem optionally generate results for benchview if exist "%BENCHVIEW_PATH%" ( call :generate_results_for_benchview || exit /b 1 - ) + ) rem Save off the results to the root directory for recovery later in Jenkins - IF EXIST "Perf-*%BENCHNAME%.xml" ( - call :run_cmd copy "Perf-*%BENCHNAME%.xml" "%CORECLR_REPO%\Perf-%BENCHNAME%-%ETW_COLLECTION%.xml" || exit /b 1 - ) - IF EXIST "Perf-*%BENCHNAME%.etl" ( - call :run_cmd copy "Perf-*%BENCHNAME%.etl" "%CORECLR_REPO%\Perf-%BENCHNAME%-%ETW_COLLECTION%.etl" || exit /b 1 + for %%e in (xml etl log) do ( + IF EXIST ".\%LV_RUNID%-%BENCHNAME%.%%e" ( + call :run_cmd xcopy /vy ".\%LV_RUNID%-%BENCHNAME%.%%e" .. || exit /b 1 + ) ) exit /b 0 @@ -219,7 +227,7 @@ rem Sets the test architecture. rem **************************************************************************** set TEST_ARCH=%TEST_ARCHITECTURE% exit /b 0 - + :verify_benchview_tools rem **************************************************************************** rem Verifies that the path to the benchview tools is correct. @@ -231,7 +239,7 @@ rem **************************************************************************** ) ) exit /b 0 - + :verify_core_overlay rem **************************************************************************** rem Verify that the Core_Root folder exist. @@ -243,7 +251,7 @@ rem **************************************************************************** ) exit /b 0 - :set_collection_config +:set_collection_config rem **************************************************************************** rem Set's the config based on the providers used for collection rem **************************************************************************** @@ -254,7 +262,7 @@ rem **************************************************************************** ) exit /b 0 - + :set_perf_run_log rem **************************************************************************** rem Sets the script's output log file. @@ -271,7 +279,7 @@ rem **************************************************************************** rem Creates the sandbox folder used by the script to copy binaries locally, rem and execute benchmarks. rem **************************************************************************** - if exist sandbox rd /s /q sandbox + if exist sandbox rmdir /s /q sandbox if exist sandbox call :print_error Failed to remove the sandbox folder& exit /b 1 if not exist sandbox mkdir sandbox if not exist sandbox call :print_error Failed to create the sandbox folder& exit /b 1 @@ -293,7 +301,7 @@ rem **************************************************************************** set LV_MEASUREMENT_ARGS=%LV_MEASUREMENT_ARGS% --append for /f %%f in ('dir /b Perf-*%BENCHNAME%.xml 2^>nul') do ( - call :run_cmd py.exe "%BENCHVIEW_PATH%\measurement.py" %LV_MEASUREMENT_ARGS% %%f + call :run_cmd py.exe "%BENCHVIEW_PATH%\measurement.py" %LV_MEASUREMENT_ARGS% %%f IF !ERRORLEVEL! NEQ 0 ( call :print_error Failed to generate BenchView measurement data. @@ -322,7 +330,7 @@ setlocal set LV_SUBMISSION_ARGS=%LV_SUBMISSION_ARGS% --machinepool "PerfSnake" call :run_cmd py.exe "%BENCHVIEW_PATH%\submission.py" measurement.json %LV_SUBMISSION_ARGS% - + IF %ERRORLEVEL% NEQ 0 ( call :print_error Creating BenchView submission data failed. exit /b 1 @@ -360,7 +368,7 @@ rem **************************************************************************** echo -collectionFlags This is used to specify what collectoin flags get passed to the performance echo harness that is doing the test running. If this is not specified we only use stopwatch. echo Other flags are "default", which is the whatever the test being run specified, "CacheMisses", - echo "BranchMispredictions", and "InstructionsRetired". + echo "BranchMispredictions", and "InstructionsRetired". exit /b %ERRORLEVEL% :print_error @@ -368,19 +376,17 @@ rem **************************************************************************** rem Function wrapper that unifies how errors are output by the script. rem Functions output to the standard error. rem **************************************************************************** - echo [%DATE%][%TIME:~0,-3%][ERROR] %* 1>&2 + call :print_to_console [ERROR] %* 1>&2 exit /b %ERRORLEVEL% :print_to_console rem **************************************************************************** -rem Sends text to the console screen, no matter what (even when the script's -rem output is redirected). This can be useful to provide information on where -rem the script is executing. +rem Sends text to the console screen. This can be useful to provide +rem information on where the script is executing. rem **************************************************************************** - if defined _debug ( - echo [%DATE%][%TIME:~0,-3%] %* >CON - ) - echo [%DATE%][%TIME:~0,-3%] %* + echo/ + echo/%USERNAME%@%COMPUTERNAME% "%CD%" + echo/[%DATE%][%TIME:~0,-3%] %* exit /b %ERRORLEVEL% :run_cmd @@ -396,9 +402,3 @@ rem **************************************************************************** call :print_to_console $ %* call %* exit /b %ERRORLEVEL% - -:skip_failures -rem **************************************************************************** -rem Skip known failures -rem **************************************************************************** - exit /b 0 diff --git a/tests/scripts/run-xunit-perf.sh b/tests/scripts/run-xunit-perf.sh index 2797174f329c..e6758e59fb0c 100755 --- a/tests/scripts/run-xunit-perf.sh +++ b/tests/scripts/run-xunit-perf.sh @@ -340,7 +340,7 @@ for testcase in ${tests[@]}; do echo "----------" echo " Running $testname" echo "----------" - run_command $stabilityPrefix ./corerun PerfHarness.dll $test --perf:runid Perf --perf:collect $collectionflags || exit 1 + run_command $stabilityPrefix ./corerun PerfHarness.dll $test --perf:runid Perf --perf:collect $collectionflags 1>"Perf-$filename.log" 2>&1 || exit 1 if [ -d "$BENCHVIEW_TOOLS_PATH" ]; then run_command python3.5 "$BENCHVIEW_TOOLS_PATH/measurement.py" xunit "Perf-$filename.xml" --better desc $hasWarmupRun --append || { echo [ERROR] Failed to generate BenchView data; @@ -349,6 +349,10 @@ for testcase in ${tests[@]}; do fi # Rename file to be archived by Jenkins. + mv -f "Perf-$filename.log" "$CORECLR_REPO/Perf-$filename-$perfCollection.log" || { + echo [ERROR] Failed to move "Perf-$filename.log" to "$CORECLR_REPO". + exit 1; + } mv -f "Perf-$filename.xml" "$CORECLR_REPO/Perf-$filename-$perfCollection.xml" || { echo [ERROR] Failed to move "Perf-$filename.xml" to "$CORECLR_REPO". exit 1; diff --git a/tests/src/Common/PerfHarness/PerfHarness.csproj b/tests/src/Common/PerfHarness/PerfHarness.csproj index a15e8022aea2..39eff59c125e 100644 --- a/tests/src/Common/PerfHarness/PerfHarness.csproj +++ b/tests/src/Common/PerfHarness/PerfHarness.csproj @@ -6,9 +6,7 @@ - - 1.0.0-beta-build0006 - + \ No newline at end of file diff --git a/tests/src/Common/external/external.depproj b/tests/src/Common/external/external.depproj index 464d27e01948..51b36208b6ae 100644 --- a/tests/src/Common/external/external.depproj +++ b/tests/src/Common/external/external.depproj @@ -22,19 +22,19 @@ 1.1.1 - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.3-alpha-experimental + $(MicrosoftDiagnosticsTracingLibraryVersion) 9.0.1 @@ -43,7 +43,7 @@ $(XunitPackageVersion) - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) $(XunitPackageVersion) diff --git a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj index 27af3866332f..9155703cd365 100644 --- a/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj +++ b/tests/src/JIT/config/benchmark+roslyn/benchmark+roslyn.csproj @@ -11,19 +11,19 @@ 1.1.1 - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.3-alpha-experimental + $(MicrosoftDiagnosticsTracingLibraryVersion) 2.0.0-preview2-25302-03 @@ -80,7 +80,7 @@ $(XunitPackageVersion) - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) $(XunitPackageVersion) diff --git a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj index 20accfe300a8..36181432d274 100644 --- a/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj +++ b/tests/src/JIT/config/benchmark+serialize/benchmark+serialize.csproj @@ -8,19 +8,19 @@ - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.3-alpha-experimental + $(MicrosoftDiagnosticsTracingLibraryVersion) 2.0.0-preview2-25302-03 @@ -80,7 +80,7 @@ $(XunitPackageVersion) - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) $(XunitPackageVersion) diff --git a/tests/src/JIT/config/benchmark/benchmark.csproj b/tests/src/JIT/config/benchmark/benchmark.csproj index e5285c1fec09..f739151181dc 100644 --- a/tests/src/JIT/config/benchmark/benchmark.csproj +++ b/tests/src/JIT/config/benchmark/benchmark.csproj @@ -8,19 +8,19 @@ - 1.0.0-beta-build0003 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0003 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0003 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0003 + $(XUnitPerformanceApiVersion) - 1.0.3-alpha-experimental + $(MicrosoftDiagnosticsTracingLibraryVersion) 2.0.0-preview2-25302-03 @@ -95,7 +95,7 @@ $(XunitPackageVersion) - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) $(XunitPackageVersion) @@ -110,5 +110,5 @@ + DependsOnTargets="ResolveReferences" /> \ No newline at end of file diff --git a/tests/src/performance/performance.csproj b/tests/src/performance/performance.csproj index 5a6ca1279fac..8133535f4514 100644 --- a/tests/src/performance/performance.csproj +++ b/tests/src/performance/performance.csproj @@ -8,19 +8,19 @@ - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.0-beta-build0006 + $(XUnitPerformanceApiVersion) - 1.0.3-alpha-experimental + $(MicrosoftDiagnosticsTracingLibraryVersion) 2.0.0-preview2-25302-03 @@ -92,7 +92,7 @@ $(XunitPackageVersion) - 1.0.2-prerelease-00177 + $(XUnitConsoleNetCoreVersion) $(XunitPackageVersion)