Update PGO counts and port infrastructure fixes for Preview3 (#12260)

Daniel Podder · web-flow · commit a0975b851ea4 · 2017-06-13T23:34:39.000-07:00
This PR is a roll-up of a handful of fixes made to the performance infrastructure and Linux PGO optimization support in master. The full list of changes follow: * Cherry-pick #12167, fixing an issue where the characters `'<'` and `'>'` broke Windows perf jobs when included in the PR title * Port #12201, revving the optdata package reference to the latest version and reflecting fixes made in the way we produce and package up counts on Linux * Cherry-pick #12238, so that we run perf tests against Linux binaries built the same way as official builds * Port #12239, fixing an issue where profile data was not being found on some Linux builds * Cherry-pick #12248, fixing a release build break on older toolsets (LLVM3.5) by disabling PGO and issuing a warning
diff --git a/dir.props b/dir.props
@@ -23,7 +23,7 @@
 
   <!-- Profile-based optimization data package versions -->
   <PropertyGroup>
-    <PgoDataPackageVersion>2.0.0-release-20170531-3000</PgoDataPackageVersion>
+    <PgoDataPackageVersion>2.0.0-release-20170612-1411</PgoDataPackageVersion>
     <!--<IbcDataPackageVersion></IbcDataPackageVersion>-->
   </PropertyGroup>
 
diff --git a/perf.groovy b/perf.groovy
@@ -88,9 +88,9 @@ def static getOSGroup(def os) {
                         batchFile("\"%WORKSPACE%\\nuget.exe\" install Microsoft.BenchView.JSONFormat -Source http://benchviewtestfeed.azurewebsites.net/nuget -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
                         //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
                         //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                        batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
-                        "set BENCHVIEWNAME=${benchViewName}\n" +
-                        "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+                        batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                        "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                        "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"%BENCHVIEWNAME%\" --user \"dotnet-bot@microsoft.com\"\n" +
                         "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
                         batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -198,9 +198,9 @@ def static getOSGroup(def os) {
                     batchFile("C:\\Tools\\nuget.exe install Microsoft.BenchView.ThroughputBenchmarks.${architecture}.${os} -Source https://dotnet.myget.org/F/dotnet-core -OutputDirectory \"%WORKSPACE%\" -Prerelease -ExcludeVersion")
                     //Do this here to remove the origin but at the front of the branch name as this is a problem for BenchView
                     //we have to do it all as one statement because cmd is called each time and we lose the set environment variable
-                    batchFile("if [%GIT_BRANCH:~0,7%] == [origin/] (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%) else (set GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%)\n" +
-                    "set BENCHVIEWNAME=${benchViewName}\n" +
-                    "set BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\n" +
+                    batchFile("if \"%GIT_BRANCH:~0,7%\" == \"origin/\" (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH:origin/=%\") else (set \"GIT_BRANCH_WITHOUT_ORIGIN=%GIT_BRANCH%\")\n" +
+                    "set \"BENCHVIEWNAME=${benchViewName}\"\n" +
+                    "set \"BENCHVIEWNAME=%BENCHVIEWNAME:\"=%\"\n" +
                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\submission-metadata.py\" --name \"${benchViewName}\" --user \"dotnet-bot@microsoft.com\"\n" +
                     "py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\build.py\" git --branch %GIT_BRANCH_WITHOUT_ORIGIN% --type ${runType}")
                     batchFile("py \"%WORKSPACE%\\Microsoft.BenchView.JSONFormat\\tools\\machinedata.py\"")
@@ -234,10 +234,32 @@ def static getOSGroup(def os) {
     }
 }
 
+def static getFullPerfJobName(def project, def os, def isPR) {
+    return Utilities.getFullJobName(project, "perf_${os}", isPR)
+}
+
 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 [true, false].each { isPR ->
-    ['Ubuntu14.04'].each { os ->
-        def newJob = job(Utilities.getFullJobName(project, "perf_${os}", isPR)) {
+    def fullBuildJobName = Utilities.getFullJobName(project, 'perf_linux_build', isPR)
+    def architecture = 'x64'
+    def configuration = 'Release'
+
+    // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+    ['RHEL7.2'].each { os ->
+        def newBuildJob = job(fullBuildJobName) {
+            steps {
+                shell("./build.sh verbose ${architecture} ${configuration}")
+            }
+        }
+        Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+        Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+        Utilities.addArchival(newBuildJob, "bin/Product/**,bin/obj/*/tests/**/*.dylib,bin/obj/*/tests/**/*.so", "bin/Product/**/.nuget/**")
+    }
+
+    // Actual perf testing on the following OSes
+    def perfOSList = ['Ubuntu14.04']
+    perfOSList.each { os ->
+        def newJob = job(getFullPerfJobName(project, os, isPR)) {
 
             label('linux_clr_perf')
             wrappers {
@@ -254,22 +276,26 @@ def static getOSGroup(def os) {
                 }
             }
 
-            // Cap the maximum number of iterations to 21.
             parameters {
+                // Cap the maximum number of iterations to 21.
                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
                 stringParam('XUNIT_PERFORMANCE_MAX_ITERATION_INNER_SPECIFIED', '21', 'Sets the number of iterations to twenty one.  We are doing this to limit the amount of data that we upload as 20 iterations is enought to get a good sample')
+                stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
             }
 
             def osGroup = getOSGroup(os)
-            def architecture = 'x64'
-            def configuration = 'Release'
             def runType = isPR ? 'private' : 'rolling'
             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 
             steps {
                 shell("./tests/scripts/perf-prep.sh")
                 shell("./init-tools.sh")
-                shell("./build.sh ${architecture} ${configuration}")
+                copyArtifacts(fullBuildJobName) {
+                    includePatterns("bin/**")
+                    buildSelector {
+                        buildNumber('\${PRODUCT_BUILD}')
+                    }
+                }
                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -306,26 +332,73 @@ def static getOSGroup(def os) {
                 numToKeep(1000)
             }
         }
+    } // os
+
+    def flowJobPerfRunList = perfOSList.collect { os ->
+        "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullPerfJobName(project, os, isPR)}') }"
+    }
+    def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_linux_flow", isPR, '')) {
         if (isPR) {
-            TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-            builder.setGithubContext("${os} Perf Tests")
-            builder.triggerOnlyOnComment()
-            builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+perf.*")
-            builder.triggerForBranch(branch)
-            builder.emitTrigger(newJob)
-        }
-        else {
-            // Set a push trigger
-            TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-            builder.emitTrigger(newJob)
+            parameters {
+                stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+            }
         }
-    } // os
+        buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+    ${flowJobPerfRunList.join(",\n    ")}
+)
+""")
+    }
+
+    Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+    Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+    if (isPR) {
+        TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+        builder.setGithubContext("Linux Perf Test Flow")
+        builder.triggerOnlyOnComment()
+        builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+perf\\W+flow.*")
+        builder.triggerForBranch(branch)
+        builder.emitTrigger(newFlowJob)
+    }
+    else {
+        // Set a push trigger
+        TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+        builder.emitTrigger(newFlowJob)
+    }
+
 } // isPR
 
+def static getFullThroughputJobName(def project, def os, def isPR) {
+    return Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)
+}
+
 // Create the Linux/OSX/CentOS coreclr test leg for debug and release and each scenario
 [true, false].each { isPR ->
-    ['Ubuntu14.04'].each { os ->
-        def newJob = job(Utilities.getFullJobName(project, "perf_throughput_${os}", isPR)) {
+    def fullBuildJobName = Utilities.getFullJobName(project, 'perf_throughput_linux_build', isPR)
+    def architecture = 'x64'
+    def configuration = 'Release'
+
+    // Build has to happen on RHEL7.2 (that's where we produce the bits we ship)
+    ['RHEL7.2'].each { os ->
+        def newBuildJob = job(fullBuildJobName) {
+            steps {
+                shell("./build.sh verbose ${architecture} ${configuration}")
+            }
+        }
+        Utilities.setMachineAffinity(newBuildJob, os, 'latest-or-auto')
+        Utilities.standardJobSetup(newBuildJob, project, isPR, "*/${branch}")
+        Utilities.addArchival(newBuildJob, "bin/Product/**")
+    }
+
+    // Actual perf testing on the following OSes
+    def throughputOSList = ['Ubuntu14.04']
+    throughputOSList.each { os ->
+        def newJob = job(getFullThroughputJobName(project, os, isPR)) {
 
             label('linux_clr_perf')
                 wrappers {
@@ -341,16 +414,24 @@ def static getOSGroup(def os) {
                     stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
                 }
             }
+
+            parameters {
+                stringParam('PRODUCT_BUILD', '', 'Build number from which to copy down the CoreCLR Product binaries built for Linux')
+            }
+
             def osGroup = getOSGroup(os)
-            def architecture = 'x64'
-            def configuration = 'Release'
             def runType = isPR ? 'private' : 'rolling'
             def benchViewName = isPR ? 'coreclr private \$BenchviewCommitName' : 'coreclr rolling \$GIT_BRANCH_WITHOUT_ORIGIN \$GIT_COMMIT'
 
             steps {
                 shell("bash ./tests/scripts/perf-prep.sh --throughput")
                 shell("./init-tools.sh")
-                shell("./build.sh ${architecture} ${configuration}")
+                copyArtifacts(fullBuildJobName) {
+                    includePatterns("bin/Product/**")
+                    buildSelector {
+                        buildNumber('\${PRODUCT_BUILD}')
+                    }
+                }
                 shell("GIT_BRANCH_WITHOUT_ORIGIN=\$(echo \$GIT_BRANCH | sed \"s/[^/]*\\/\\(.*\\)/\\1 /\")\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/submission-metadata.py\" --name \" ${benchViewName} \" --user \"dotnet-bot@microsoft.com\"\n" +
                 "python3.5 \"\${WORKSPACE}/tests/scripts/Microsoft.BenchView.JSONFormat/tools/build.py\" git --branch \$GIT_BRANCH_WITHOUT_ORIGIN --type ${runType}")
@@ -383,18 +464,43 @@ def static getOSGroup(def os) {
                 numToKeep(1000)
             }
         }
+    } // os
+
+    def flowJobTPRunList = throughputOSList.collect { os ->
+        "{ build(params + [PRODUCT_BUILD: b.build.number], '${getFullThroughputJobName(project, os, isPR)}') }"
+    }
+    def newFlowJob = buildFlowJob(Utilities.getFullJobName(project, "perf_throughput_linux_flow", isPR, '')) {
         if (isPR) {
-            TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
-            builder.setGithubContext("${os} Throughput Perf Tests")
-            builder.triggerOnlyOnComment()
-            builder.setCustomTriggerPhrase("(?i).*test\\W+${os}\\W+throughput.*")
-            builder.triggerForBranch(branch)
-            builder.emitTrigger(newJob)
-        }
-        else {
-            // Set a push trigger
-            TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
-            builder.emitTrigger(newJob)
+            parameters {
+                stringParam('BenchviewCommitName', '\${ghprbPullTitle}', 'The name that you will be used to build the full title of a run in Benchview.  The final name will be of the form <branch> private BenchviewCommitName')
+            }
         }
-    } // os
+        buildFlow("""
+// First, build the bits on RHEL7.2
+b = build(params, '${fullBuildJobName}')
+
+// Then, run the perf tests
+parallel(
+    ${flowJobTPRunList.join(",\n    ")}
+)
+""")
+    }
+
+    Utilities.setMachineAffinity(newFlowJob, 'Windows_NT', 'latest-or-auto')
+    Utilities.standardJobSetup(newFlowJob, project, isPR, "*/${branch}")
+
+    if (isPR) {
+        TriggerBuilder builder = TriggerBuilder.triggerOnPullRequest()
+        builder.setGithubContext("Linux Throughput Perf Test Flow")
+        builder.triggerOnlyOnComment()
+        builder.setCustomTriggerPhrase("(?i).*test\\W+linux\\W+throughput\\W+flow.*")
+        builder.triggerForBranch(branch)
+        builder.emitTrigger(newFlowJob)
+    }
+    else {
+        // Set a push trigger
+        TriggerBuilder builder = TriggerBuilder.triggerOnCommit()
+        builder.emitTrigger(newFlowJob)
+    }
+
 } // isPR
diff --git a/pgosupport.cmake b/pgosupport.cmake
@@ -11,17 +11,18 @@ function(add_pgo TargetName)
     if(WIN32)
         set(ProfileFileName "${TargetName}.pgd")
     else(WIN32)
-        # Clang/LLVM uses one profdata file for the entire repo
-        set(ProfileFileName "coreclr.profdata")
+        set(ProfileFileName "${TargetName}.profdata")
     endif(WIN32)
 
     set(CLR_CMAKE_OPTDATA_PACKAGEWITHRID "optimization.${CLR_CMAKE_TARGET_OS}-${CLR_CMAKE_TARGET_ARCH}.PGO.CoreCLR")
+
+    # On case-sensitive file systems, NuGet packages are restored to lowercase paths
+    string(TOLOWER "${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}" OptDataVersionedSubPath)
+
     file(TO_NATIVE_PATH
-        "${CLR_CMAKE_PACKAGES_DIR}/${CLR_CMAKE_OPTDATA_PACKAGEWITHRID}/${CLR_CMAKE_OPTDATA_VERSION}/data/${ProfileFileName}"
+        "${CLR_CMAKE_PACKAGES_DIR}/${OptDataVersionedSubPath}/data/${ProfileFileName}"
         ProfilePath
     )
-    # NuGet packages are restored to lowercase paths
-    string(TOLOWER "${ProfilePath}" ProfilePath)
 
     if(CLR_CMAKE_PGO_INSTRUMENT)
         if(WIN32)
@@ -41,10 +42,16 @@ function(add_pgo TargetName)
                 set_property(TARGET ${TargetName} APPEND_STRING PROPERTY LINK_FLAGS_RELWITHDEBINFO " /LTCG /USEPROFILE:PGD=${ProfilePath}")
             else(WIN32)
                 if(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)
-                    if(HAVE_LTO)
-                        target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath} -Wno-profile-instr-out-of-date)
-                        set_property(TARGET ${TargetName} APPEND_STRING PROPERTY LINK_FLAGS " -flto -fuse-ld=gold -fprofile-instr-use=${ProfilePath}")
-                    endif(HAVE_LTO)
+                    if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6)
+                        if(HAVE_LTO)
+                            target_compile_options(${TargetName} PRIVATE -flto -fprofile-instr-use=${ProfilePath} -Wno-profile-instr-out-of-date)
+                            set_property(TARGET ${TargetName} APPEND_STRING PROPERTY LINK_FLAGS " -flto -fuse-ld=gold -fprofile-instr-use=${ProfilePath}")
+                        else(HAVE_LTO)
+                            message(WARNING "LTO is not supported, skipping profile guided optimizations")
+                        endif(HAVE_LTO)
+                    else(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6)
+                        message(WARNING "PGO is not supported; Clang 3.6 or later is required for profile guided optimizations")
+                    endif(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.6)
                 endif(UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELEASE OR UPPERCASE_CMAKE_BUILD_TYPE STREQUAL RELWITHDEBINFO)
             endif(WIN32)
         endif(EXISTS ${ProfilePath})
