From 3e6fe517467902a0aba3d47ef528fc1bcafcee3d Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 29 Jun 2018 16:23:49 -0700 Subject: [PATCH 1/8] Modify netci.groovy --- netci.groovy | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/netci.groovy b/netci.groovy index 11aa4a5484df..9ddec82d307f 100755 --- a/netci.groovy +++ b/netci.groovy @@ -2218,14 +2218,23 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}" if (doCoreFxTesting) { + assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') if (scenario == 'corefx_innerloop') { - // Create CORE_ROOT and testhost - buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly" - buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" - - // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") - Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") + if (os == 'Windows_NT') { + // Create CORE_ROOT and testhost + buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly" + buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" + + // TODO: Move outside of the if block, once logic Unix PR Logic is merged + // Archive and process (only) the test results + Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") + Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") + } + else if (os == 'Ubuntu' || os == 'OSX10.12') { + buildCommands += "./build.sh ${lowerConfiguration} ${arch} skiptests" + buildCommands += "./build-test.sh ${lowerConfiguration} ${arch} generatetesthostonly" + buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=%WORKSPACE%/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=%WORKSPACE%" + } } else { def workspaceRelativeFxRoot = "_/fx" @@ -2808,7 +2817,7 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi } break case 'corefx_innerloop': - if (os != 'Windows_NT'|| architecture != 'x64') { + if (os != 'Windows_NT' || os != 'Ubuntu' || os != 'OSX10.12' || architecture != 'x64') { return false } if(configuration != 'Release' && configuration != 'Checked') { From 9d6de388c0e0a4c0ba0427c39c3cfeacb0ecd5c0 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Fri, 29 Jun 2018 19:19:20 -0700 Subject: [PATCH 2/8] Add script exit codes --- build-test.sh | 5 +++++ tests/runtest.sh | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/build-test.sh b/build-test.sh index 5339dc1116c7..b5badd09ab84 100755 --- a/build-test.sh +++ b/build-test.sh @@ -384,6 +384,7 @@ usage() echo "verbose - optional argument to enable verbose build output." echo "rebuild - if tests have already been built - rebuild them" echo "generatelayoutonly - only pull down dependencies and build coreroot" + echo "generatetesthostonly - only pull down dependencies and build coreroot and the CoreFX testhost" echo "buildagainstpackages - pull down and build using packages." echo "runtests - run tests after building them" echo "ziptests - zips CoreCLR tests & Core_Root for a Helix run" @@ -641,6 +642,10 @@ while :; do __GenerateLayoutOnly=1 ;; + generatetesthostonly) + exit 0 + ;; + buildagainstpackages) __BuildAgainstPackagesArg=1 ;; diff --git a/tests/runtest.sh b/tests/runtest.sh index c63ed4a30bb3..58038f35f45a 100755 --- a/tests/runtest.sh +++ b/tests/runtest.sh @@ -67,6 +67,10 @@ function print_usage { echo ' is zero when launching this script. This option is intended for use in CI.' echo ' --xunitOutputPath= : Create xUnit XML report at the specifed path (default: /coreclrtests.xml)' echo '' + echo 'CoreFX Test Options ' + echo ' --corefxtests : Runs CoreFX tests' + echo ' --corefxtestsall : Runs all available CoreFX tests' + echo '' echo 'Runtime Code Coverage options:' echo ' --coreclr-coverage : Optional argument to get coreclr code coverage reports' echo ' --coreclr-objs= : Location of root of the object directory' @@ -1168,6 +1172,12 @@ do --runcrossgentests) export RunCrossGen=1 ;; + --corefxtests) + exit 0 + ;; + --corefxtestsall) + exit 0 + ;; --sequential) ((maxProcesses = 1)) ;; From 34a1eda2692e4cb8ec0bea16bfbde6ecb45e938a Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Mon, 2 Jul 2018 16:35:37 -0700 Subject: [PATCH 3/8] Ad PR Triggers for Ubuntu and OSX10.12 --- netci.groovy | 99 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 62 insertions(+), 37 deletions(-) diff --git a/netci.groovy b/netci.groovy index 9ddec82d307f..e4266f73a0c7 100755 --- a/netci.groovy +++ b/netci.groovy @@ -1565,6 +1565,14 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*") break } + + else if (scenario == 'corefx_innerloop') { + if (configuration == 'Release' || configuration == 'Checked') { + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") + } + break + } + // fall through case 'OSX10.12': @@ -1626,6 +1634,12 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, } break + case 'corefx_innerloop': + if (configuration == 'Release' || configuration == 'Checked') { + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") + } + break + default: if (isJitStressScenario(scenario)) { def displayStr = getStressModeDisplayName(scenario) @@ -2218,23 +2232,15 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR runtestArguments = "${lowerConfiguration} ${arch} ${testOpts}" if (doCoreFxTesting) { - assert (os == 'Ubuntu' || os == 'Windows_NT' || os == 'OSX10.12') if (scenario == 'corefx_innerloop') { - if (os == 'Windows_NT') { - // Create CORE_ROOT and testhost - buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly" - buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" - - // TODO: Move outside of the if block, once logic Unix PR Logic is merged - // Archive and process (only) the test results - Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") - Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") - } - else if (os == 'Ubuntu' || os == 'OSX10.12') { - buildCommands += "./build.sh ${lowerConfiguration} ${arch} skiptests" - buildCommands += "./build-test.sh ${lowerConfiguration} ${arch} generatetesthostonly" - buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=%WORKSPACE%/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=%WORKSPACE%" - } + // Create CORE_ROOT and testhost + buildCommands += "build-test.cmd ${lowerConfiguration} ${arch} buildtesthostonly" + buildCommands += "tests\\runtest.cmd ${runtestArguments} CoreFXTestsAll" + + // Archive and process (only) the test results + Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") + Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") + } else { def workspaceRelativeFxRoot = "_/fx" @@ -2422,32 +2428,51 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR Utilities.addXUnitDotNETResults(newJob, '**/pal_tests.xml') } else { - // Corefx stress testing - assert os == 'Ubuntu' - assert architecture == 'x64' - assert lowerConfiguration == 'checked' - assert isJitStressScenario(scenario) + if(scenario == 'corefx_innerloop') { - // Build coreclr - buildCommands += "./build.sh ${lowerConfiguration} ${architecture}" + assert os == 'Ubuntu' + assert architecture == 'x64' + assert lowerConfiguration == 'checked' || lowerConfiguration == 'release' + assert isJitStressScenario(scenario) - def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" + buildCommands += "./build.sh ${lowerConfiguration} ${arch} skiptests" + buildCommands += "./build-test.sh ${lowerConfiguration} ${arch} generatetesthostonly" + buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=%WORKSPACE%/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=%WORKSPACE%" + + break + // TODO - uncomment once https://github.com/dotnet/coreclr/pull/18753 has been merged + // Archive and process (only) the test results + // Utilities.addArchival(newJob, "bin/Logs/**/testResults.xml") + // Utilities.addXUnitDotNETResults(newJob, "bin/Logs/**/testResults.xml") + } + else { + // Corefx stress testing + assert os == 'Ubuntu' + assert architecture == 'x64' + assert lowerConfiguration == 'checked' + assert isJitStressScenario(scenario) - def envScriptCmds = envScriptCreate(os, scriptFileName) - envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) - envScriptCmds += envScriptFinalize(os, scriptFileName) - buildCommands += envScriptCmds + // Build coreclr + buildCommands += "./build.sh ${lowerConfiguration} ${architecture}" - // Build and text corefx - def workspaceRelativeFxRoot = "_/fx" - def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}" - def fxBranch = getFxBranch(branch) + def scriptFileName = "\$WORKSPACE/set_stress_test_env.sh" - buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}" + def envScriptCmds = envScriptCreate(os, scriptFileName) + envScriptCmds += envScriptSetStressModeVariables(os, Constants.jitStressModeScenarios[scenario], scriptFileName) + envScriptCmds += envScriptFinalize(os, scriptFileName) + buildCommands += envScriptCmds - // Archive and process (only) the test results - Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml") - Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml") + // Build and text corefx + def workspaceRelativeFxRoot = "_/fx" + def absoluteFxRoot = "\$WORKSPACE/${workspaceRelativeFxRoot}" + def fxBranch = getFxBranch(branch) + + buildCommands += "python -u \$WORKSPACE/tests/scripts/run-corefx-tests.py -arch ${architecture} -ci_arch ${architecture} -build_type ${configuration} -fx_root ${absoluteFxRoot} -fx_branch ${fxBranch} -env_script ${scriptFileName}" + + // Archive and process (only) the test results + Utilities.addArchival(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml") + Utilities.addXUnitDotNETResults(newJob, "${workspaceRelativeFxRoot}/bin/**/testResults.xml") + } } break case 'armem': @@ -2817,7 +2842,7 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi } break case 'corefx_innerloop': - if (os != 'Windows_NT' || os != 'Ubuntu' || os != 'OSX10.12' || architecture != 'x64') { + if ( (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') || architecture != 'x64') { return false } if(configuration != 'Release' && configuration != 'Checked') { From 03865ddc72242da6a646264191caa645afbbb3e3 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Tue, 3 Jul 2018 15:53:17 -0700 Subject: [PATCH 4/8] Remove release PR triggers --- netci.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netci.groovy b/netci.groovy index e4266f73a0c7..603a8f9665b3 100755 --- a/netci.groovy +++ b/netci.groovy @@ -1567,7 +1567,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, } else if (scenario == 'corefx_innerloop') { - if (configuration == 'Release' || configuration == 'Checked') { + if (configuration == 'Checked') { Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") } break @@ -1635,7 +1635,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, break case 'corefx_innerloop': - if (configuration == 'Release' || configuration == 'Checked') { + if (configuration == 'Checked') { Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") } break @@ -1757,7 +1757,7 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} via ILLink", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+${scenario}.*") break case 'corefx_innerloop': - if (configuration == 'Release' || configuration == 'Checked') { + if (configuration == 'Checked') { Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") } break From 8ce4b695c7b0855d0fd695082e7ea52c48955a13 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 4 Jul 2018 14:59:59 -0700 Subject: [PATCH 5/8] Rename arch Remove IsJitStressTestScenario assert --- netci.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/netci.groovy b/netci.groovy index 603a8f9665b3..f76a96f7eeda 100755 --- a/netci.groovy +++ b/netci.groovy @@ -2429,14 +2429,12 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR } else { if(scenario == 'corefx_innerloop') { - assert os == 'Ubuntu' assert architecture == 'x64' assert lowerConfiguration == 'checked' || lowerConfiguration == 'release' - assert isJitStressScenario(scenario) - buildCommands += "./build.sh ${lowerConfiguration} ${arch} skiptests" - buildCommands += "./build-test.sh ${lowerConfiguration} ${arch} generatetesthostonly" + buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" + buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=%WORKSPACE%/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=%WORKSPACE%" break From 6e102d9d7f50503ea49d97e5073613093ab2a558 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Wed, 4 Jul 2018 15:13:06 -0700 Subject: [PATCH 6/8] Add correct asserts --- netci.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/netci.groovy b/netci.groovy index f76a96f7eeda..981e45d884d7 100755 --- a/netci.groovy +++ b/netci.groovy @@ -2429,9 +2429,9 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR } else { if(scenario == 'corefx_innerloop') { - assert os == 'Ubuntu' + assert os == 'Ubuntu' || 'OSX10.12' assert architecture == 'x64' - assert lowerConfiguration == 'checked' || lowerConfiguration == 'release' + assert lowerConfiguration == 'checked' buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" @@ -2843,7 +2843,7 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi if ( (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') || architecture != 'x64') { return false } - if(configuration != 'Release' && configuration != 'Checked') { + if(configuration != 'Checked') { return false } break From 06246cf162320acfba20123e79212886da16556a Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Thu, 5 Jul 2018 11:11:27 -0700 Subject: [PATCH 7/8] Reformat conditions and add triggers for Release/Checked/Debug --- netci.groovy | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/netci.groovy b/netci.groovy index 981e45d884d7..fd384323d9ea 100755 --- a/netci.groovy +++ b/netci.groovy @@ -1570,6 +1570,9 @@ def static addTriggers(def job, def branch, def isPR, def architecture, def os, if (configuration == 'Checked') { Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests") } + else { + Utilities.addGithubPRTriggerForBranch(job, branch, "${os} ${architecture} ${configuration} CoreFX Tests", "(?i).*test\\W+${os}\\W+${architecture}\\W+${configuration}\\W+CoreFX Tests.*") + } break } @@ -2431,7 +2434,6 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR if(scenario == 'corefx_innerloop') { assert os == 'Ubuntu' || 'OSX10.12' assert architecture == 'x64' - assert lowerConfiguration == 'checked' buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" @@ -2840,10 +2842,10 @@ def static shouldGenerateJob(def scenario, def isPR, def architecture, def confi } break case 'corefx_innerloop': - if ( (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') || architecture != 'x64') { + if (os != 'Windows_NT' && os != 'Ubuntu' && os != 'OSX10.12') { return false } - if(configuration != 'Checked') { + if (architecture != 'x64') { return false } break From 428512da4dcf6c6c53f6818e2ce5e53c7d0fd788 Mon Sep 17 00:00:00 2001 From: Andon Andonov Date: Mon, 9 Jul 2018 10:35:12 -0700 Subject: [PATCH 8/8] Change %WORKSPACE% to \${WORKSPACE} --- netci.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netci.groovy b/netci.groovy index fd384323d9ea..fae547f5696d 100755 --- a/netci.groovy +++ b/netci.groovy @@ -2437,7 +2437,7 @@ def static calculateBuildCommands(def newJob, def scenario, def branch, def isPR buildCommands += "./build.sh ${lowerConfiguration} ${architecture} skiptests" buildCommands += "./build-test.sh ${lowerConfiguration} ${architecture} generatetesthostonly" - buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=%WORKSPACE%/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=%WORKSPACE%" + buildCommands += "./tests/runtest.sh --corefxtestsall --testHostDir=\${WORKSPACE}/bin/tests/${osGroup}.${architecture}.${configuration}/testhost/ --coreclr-src=\${WORKSPACE}" break // TODO - uncomment once https://github.com/dotnet/coreclr/pull/18753 has been merged