[Arm64] Switch to new pipelines.groovy CI (#28280)

* [Arm64] Stop running Ubuntu16.04 arm64 Debug Build on PR

* [Arm64] Switch to new pipelines.groovy CI

Author: sdmaclea

buildpipeline/linux.arm64.groovy

@@ -0,0 +1,50 @@
+@Library('dotnet-ci') _
+
+// Incoming parameters.  Access with "params.<param name>".
+// Note that the parameters will be set as env variables so we cannot use names that conflict
+// with the engineering system parameter names.
+// CGroup - Build configuration.
+// TestOuter - If true, runs outerloop, if false runs just innerloop
+
+def submittedHelixJson = null
+
+simpleDockerNode('microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180315221921') {
+    stage ('Checkout source') {
+        checkoutRepo()
+    }
+
+    def logFolder = getLogFolder()
+
+    stage ('Initialize tools') {
+        // Init tools
+        sh './init-tools.sh'
+    }
+    stage ('Generate version assets') {
+        // Generate the version assets.  Do we need to even do this for non-official builds?
+        sh "./build-managed.sh -- /t:GenerateVersionSourceFile /p:GenerateVersionSourceFile=true"
+    }
+    stage ('Sync') {
+        sh "./sync.sh -p -BuildTests=false -- /p:ArchGroup=arm64"
+    }
+    // For arm64 cross builds we split the 'Build Product' build.sh command into 3 separate parts
+    stage ('Build Native') {
+        sh """
+            export ROOTFS_DIR=/crossrootfs/arm64
+            ./build-native.sh -buildArch=arm64 -${params.CGroup}
+        """
+    }
+    stage ('Build Managed') {
+        // Cross build builds Linux Managed components using x64 target
+        // We do not want x64 packages
+        sh "./build-managed.sh -BuildPackages=false -buildArch=x64 -${params.CGroup}"
+    }
+    stage ('Build Packages') {
+        sh "./build-packages.sh -buildArch=arm64 -${params.CGroup}"
+    }
+
+    // TODO: Build Tests for arm64 when possible
+
+    // TODO: Add submission for Helix testing once we have queue for arm64 Linux working
+}
+
+// TODO: Add "Execute tests" stage once we have queue for arm64 Linux working

buildpipeline/pipelinejobs.groovy

@@ -15,13 +15,15 @@ def branch = GithubBranchName
 // **************************
 
 def linPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/linux.groovy')
+def linArm64Pipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/linux.arm64.groovy')
 def centos6Pipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/centos.6.groovy')
 def alpine36Pipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/alpine.3.6.groovy')
 def osxPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/osx.groovy')
 def winPipeline = Pipeline.createPipelineForGithub(this, project, branch, 'buildpipeline/windows.groovy')
 
 def configurations = [
     ['TGroup':"netcoreapp", 'Pipeline':linPipeline, 'Name':'Linux' ,'ForPR':"Release-x64", 'Arch':['x64']],
+    ['TGroup':"netcoreapp", 'Pipeline':linArm64Pipeline, 'Name':'Linux' ,'ForPR':"Release-arm64", 'Arch':['arm64']],
     ['TGroup':"netcoreapp", 'Pipeline':centos6Pipeline, 'Name':'CentOS.6' ,'ForPR':"", 'Arch':['x64']],
     ['TGroup':"netcoreapp", 'Pipeline':alpine36Pipeline, 'Name':'Alpine.3.6' ,'ForPR':"Debug-x64", 'Arch':['x64']],
     ['TGroup':"netcoreapp", 'Pipeline':osxPipeline, 'Name':'OSX', 'ForPR':"Debug-x64", 'Arch':['x64']],

cross/arm64_ci_script.sh

@@ -345,68 +345,6 @@ def targetGroupOsMapInnerloop = ['netcoreapp': ['Windows_NT', 'Ubuntu14.04', 'Ub
     } // targetGroup
 } // isPR
 
-// **************************
-// Define Linux ARM64 cross builds. These jobs run on every merge.
-// Some jobs run on every PR. The ones that don't run per PR can be requested via a phrase.
-// **************************
-[true, false].each { isPR ->
-    ['netcoreapp'].each { targetGroup ->
-        ['Debug', 'Release'].each { configurationGroup ->
-            def osGroup = "Linux"
-            def archGroup = 'arm64'
-            def osName = 'Ubuntu16.04'
-
-            def newJobName = "${osName.toLowerCase()}_arm64_cross_${configurationGroup.toLowerCase()}"
-
-            def newJob = job(Utilities.getFullJobName(project, newJobName, isPR)) {
-                steps {
-                    // Call the arm64_ci_script.sh script to perform the cross build of native corefx
-                    def script = "./cross/arm64_ci_script.sh --buildConfig=${configurationGroup.toLowerCase()}"
-                    shell(script)
-
-                    // Tar up the appropriate bits.
-                    shell("tar -czf bin/build.tar.gz --directory=\"bin/Linux.${archGroup}.${configurationGroup}/native\" .")
-                }
-            }
-
-            // The cross build jobs run on Ubuntu. The arm-cross-latest version
-            // contains the packages needed for cross building corefx
-            Utilities.setMachineAffinity(newJob, 'Ubuntu14.04', 'arm-cross-latest')
-
-            // Set up standard options.
-            Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}")
-
-            // Add archival for the built binaries
-            def archiveContents = "bin/build.tar.gz"
-            Utilities.addArchival(newJob, archiveContents)
-
-            newJob.with {
-                publishers {
-                    azureVMAgentPostBuildAction {
-                        agentPostBuildAction('Delete agent after build execution (when idle).')
-                    }
-                }
-            }
-
-            // Set up triggers
-            if (isPR) {
-                if (configurationGroup == "Release") {
-                    // Run Arm64 Linux Release job automatically for PR builds
-                    Utilities.addGithubPRTriggerForBranch(newJob, branch, "${osName} arm64 ${configurationGroup} Build")
-                }
-                else {
-                    // Add Arm64 Linux Debug job hook
-                    Utilities.addGithubPRTriggerForBranch(newJob, branch, "${osName} arm64 ${configurationGroup} Build", "(?i).*test\\W+${osName}\\W+arm64\\W+${configurationGroup}.*")
-                }
-            }
-            else {
-                // Set a push trigger
-                Utilities.addGithubPushTrigger(newJob)
-            }
-        } // configurationGroup
-    } // targetGroup
-} // isPR
-
 // **************************
 // Define Linux x86 builds. These jobs run daily and results will be used for CoreCLR test
 // TODO: innerloop & outerloop testing & merge to general job generation routine