Port buildpipeline json to Azure devops Yaml (#4860)

* Port buildpipeline json to Azure devops Yaml.

* Update per review comments

* few more updates to the review comments

* Fix PR build issue for Windows leg

* NetCorePublic-Int-Pool machine pool doesn't have VS 2015 or VS 2017 installed. Switching to Hosted VS2017

DotnetCLIVersion.txt

@@ -1 +1 @@
-2.1.301
+2.1.401

azure-pipelines.yml

@@ -0,0 +1,189 @@
+trigger:
+- master
+
+variables:
+  - name: TeamName
+    value: dotnet-core-acquisition
+  # Skip Running CI tests
+  - name: SkipTests
+    value: false
+  # Set build as stable to remove build number from package names, used for milestone builds
+  - name: IsStable
+    value: false
+  # Set Official Build Id
+  - name: OfficialBuildId
+    value: $(Build.BuildNumber)
+  # Produce Test build for PR and Public builds
+  - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: SignType
+      value: test
+  # Set variables only for Official build from internal
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - name: SignType
+      value: real
+  # Variable groups
+    - group: DotNet-Blob-Feed
+    - group: DotNet-Symbol-Server-Pats
+    # Used for publishing individual leg assets to azure blobstorage
+    - group: DotNet-DotNetCli-Storage
+    # Used for publishing to myget
+    - group: DotNet-MyGet-Publish
+    # Used for dotnet/versions update
+    - group: DotNet-Versions-Publish
+
+    # BlobFeed update
+    - name: _PublishBlobFeedUrl
+      value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+    - name: _BlobFeedArgs
+      value: /p:PackagesUrl=$(_PublishBlobFeedUrl)
+        /p:SymbolPackagesUrl=$(_PublishBlobFeedUrl)
+        /p:TransportFeedAccessToken=$(dotnetfeed-storage-access-key-1)
+
+    # Symbol Server update
+    - name: _SymbolServerPath
+      value: https://microsoftpublicsymbols.artifacts.visualstudio.com/DefaultCollection
+    - name: _SymbolServerArgs
+      value: /p:SymbolServerPath=$(_SymbolServerPath)
+         /p:SymbolServerPAT=$(microsoft-symbol-server-pat)
+         /p:SymbolExpirationInDays=365
+
+    # ******** Official values *****
+    - name: _CommonPublishArgs
+      value: /p:AzureAccountName=dotnetcli
+        /p:ContainerName=dotnet
+        /p:AzureAccessToken=$(dotnetcli-storage-key)
+        /p:ChecksumAzureAccountName=dotnetclichecksums
+        /p:ChecksumContainerName=dotnet
+        /p:ChecksumAzureAccessToken=$(dotnetclichecksums-storage-key)
+    # Used for publishing to dotnet myget account
+    - name: MyGetApiKey
+      value: $(dotnet-myget-org-api-key )
+    - name: MyGetFeedUrl
+      value: https://dotnet.myget.org/F/dotnet-core/api/v2/package
+    - name: MyGetSymbolsFeedUrl
+      value: https://dotnet.myget.org/F/dotnet-core/symbols/api/v2/package
+    - name: _NugetFeedArgs
+      value: /p:NuGetFeedUrl=$(MyGetFeedUrl)
+        /p:NuGetSymbolsFeedUrl=$(MyGetSymbolsFeedUrl)
+        /p:NuGetApiKey=$(MyGetApiKey)
+
+    # ******* End Dev testing Configuration ********
+jobs:
+  ################################################################################
+  # Build Linux legs
+  ################################################################################
+- template: /eng/jobs/linux-build.yml
+  parameters:
+    displayName: Build_Linux_x64_glibc
+    dockerImage: microsoft/dotnet-buildtools-prereqs:centos-7-d485f41-20173404063424
+    packageDistroListDeb: [debian.8,debian.9,ubuntu.16.04,ubuntu.18.04]
+    packageDistroListRpm: [centos.7,fedora.27,opensuse.42,oraclelinux.7,sles.12]
+    portableBuild: true
+    targetArchitecture: x64
+
+- template: /eng/jobs/linux-build.yml
+  parameters:
+    crossBuild: true
+    displayName: Build_Linux_Arm
+    dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-14.04-cross-e435274-20180323032140
+    additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/arm
+    portableBuild: true
+    skipTests: true
+    targetArchitecture: arm
+
+- template: /eng/jobs/linux-build.yml
+  parameters:
+    crossBuild: true
+    displayName: Build_Linux_Arm64
+    dockerImage: microsoft/dotnet-buildtools-prereqs:ubuntu-16.04-cross-arm64-a3ae44b-20180316023254
+    additionalRunArgs: -e ROOTFS_DIR=/crossrootfs/arm64
+    portableBuild: true
+    skipTests: true
+    targetArchitecture: arm64
+
+- template: /eng/jobs/linux-build.yml
+  parameters:
+    additionalMSBuildArgs: /p:OutputRid=linux-musl-x64
+    displayName: Build_Linux_x64_Alpine36
+    dockerImage: microsoft/dotnet-buildtools-prereqs:alpine-3.6-3148f11-20171119021156
+    portableBuild: false
+    targetArchitecture: x64
+
+- template: /eng/jobs/linux-build.yml
+  parameters:
+    additionalMSBuildArgs: /p:OutputRid=rhel.6-x64
+    displayName: Build_Linux_x64_Rhel6
+    dockerImage: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331
+    portableBuild: false
+    targetArchitecture: x64
+
+# - template: /eng/jobs/linux-build.yml
+#   parameters:
+#     additionalMSBuildArgs: /p:OutputRid=rhel.6-x64
+#     displayName: Build_Linux_x64_FreeBSD
+#     dockerImage: microsoft/dotnet-buildtools-prereqs:centos-6-376e1a3-20174311014331
+#     osGroup: FreeBSD
+#     portableBuild: true
+#     skipTests: true
+#     targetArchitecture: x64
+
+  ################################################################################
+  # Build Windows legs
+  ################################################################################
+
+# Windows x64
+- template: /eng/jobs/windows-build.yml
+  parameters:
+    buildFullPlatformManifest: true
+    displayName: Build_Windows_x64
+    publishRidAgnosticPackages: true
+    targetArchitecture: x64
+
+# Windows x86
+- template: /eng/jobs/windows-build.yml
+  parameters:
+    displayName: Build_Windows_x86
+    targetArchitecture: x86
+
+# Windows Arm64
+- template: /eng/jobs/windows-build.yml
+  parameters:
+    displayName: Build_Windows_Arm64
+    skipTests: true
+    targetArchitecture: arm64
+
+# Windows Arm
+- template: /eng/jobs/windows-build.yml
+  parameters:
+    displayName: Build_Windows_Arm
+    skipTests: true
+    targetArchitecture: arm
+
+  ################################################################################
+  # Build Build_OSX leg
+  ################################################################################
+- template: /eng/jobs/osx-build.yml
+
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  ################################################################################
+  # Signing validation
+  ################################################################################
+  - template: /eng/jobs/validate-signing.yml
+
+  ################################################################################
+  # Finalize build publish
+  # publish nuget packages to blobfeed, symbolserver, and myget
+  ################################################################################
+  - template: /eng/jobs/finalize-publish.yml
+    parameters:
+      _PublishType: nopublishtype
+
+  ################################################################################
+  # Publish to B.A.R
+  ################################################################################
+  - template: /eng/common/templates/job/publish-build-assets.yml
+    parameters:
+      pool:
+        name: dotnet-internal-temp
+      dependsOn:
+        - Finalize_Publish

config.json

@@ -196,12 +196,42 @@
             "Project": "src\\corehost\\build.proj"
           }
         },
+        "packaging": {
+          "description": "Build packaging project.",
+          "settings": {
+            "Project": "src\\pkg\\packaging\\dir.proj"
+          }
+        },
+        "pkg": {
+          "description": "Build nuget packaging project.",
+          "settings": {
+            "Project": "src\\pkg\\dir.proj"
+          }
+        },
+        "publishing": {
+          "description": "Build publishing project.",
+          "settings": {
+            "Project": "publish\\publish.proj"
+          }
+        },
+        "sharedfx": {
+          "description": "Build Shared Framework project.",
+          "settings": {
+            "Project": "src\\sharedframework\\sharedframework.proj"
+          }
+        },
         "signing-validation": {
           "description": "Build signing validation project.",
           "settings": {
             "Project": "signing\\signingvalidation.proj"
           }
         },
+        "signing": {
+          "description": "Build signing project.",
+          "settings": {
+            "Project": "signing\\sign.proj"
+          }
+        },
         "GenerateVersion": {
           "description": "Generates the version header for native binaries.",
           "settings": {
@@ -345,12 +375,12 @@
     "msbuild": {
       "osSpecific":{
         "windows": {
-          "defaultParameters": "/nologo /verbosity:minimal /clp:Summary /maxcpucount /nodeReuse:false",
-          "path": "Tools/msbuild.cmd"
+          "defaultParameters": "msbuild /nologo /verbosity:minimal /clp:Summary /maxcpucount /nodeReuse:false",
+          "path": "Tools/dotnetcli/dotnet"
         },
         "unix":{
-          "defaultParameters": "/nologo /verbosity:minimal /clp:Summary /maxcpucount /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log",
-          "path": "Tools/msbuild.sh"
+          "defaultParameters": "msbuild /nologo /verbosity:minimal /clp:Summary /maxcpucount /l:BinClashLogger,Tools/Microsoft.DotNet.Build.Tasks.dll;LogFile=binclash.log",
+          "path": "Tools/dotnetcli/dotnet"
         }
       },
       "valueTypes": {

dependencies.props

@@ -45,13 +45,13 @@
   <!-- Package versions used as toolsets -->
   <PropertyGroup>
     <FeedTasksPackage>Microsoft.DotNet.Build.Tasks.Feed</FeedTasksPackage>
-    <FeedTasksPackageVersion>3.0.0-preview1-03430-01</FeedTasksPackageVersion>
+    <FeedTasksPackageVersion>2.2.0-beta.18578.9</FeedTasksPackageVersion>
   </PropertyGroup>
 
   <!-- Publish symbol build task package -->
   <PropertyGroup>
     <PublishSymbolsPackage>Microsoft.SymbolUploader.Build.Task</PublishSymbolsPackage>
-    <PublishSymbolsPackageVersion>1.0.0-beta-63412-03</PublishSymbolsPackageVersion>
+    <PublishSymbolsPackageVersion>1.0.0-beta-63604-05</PublishSymbolsPackageVersion>
   </PropertyGroup>
 
   <ItemGroup>

eng/jobs/finalize-publish.yml

@@ -0,0 +1,111 @@
+parameters:
+  _PublishType: {}
+jobs:
+  - job: Finalize_Publish
+    displayName: Finalize_Publish
+    # Run only if all build legs succeeded
+    condition: and(succeeded('Build_Linux_x64_glibc'),
+                  succeeded('Build_Linux_Arm64'),
+                  succeeded('Build_Linux_Arm'),
+                  succeeded('Build_Linux_x64_Alpine36'),
+                  succeeded('Build_Linux_x64_Rhel6'),
+                  succeeded('Build_OSX'),
+                  succeeded('Build_Windows_x64'),
+                  succeeded('Build_Windows_x86'),
+                  succeeded('Build_Windows_Arm'),
+                  succeeded('Build_Windows_Arm64'),
+                  ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))
+    # Run after all dependent legs are executed
+    dependsOn: 
+      - Build_Linux_x64_glibc
+      - Build_Linux_Arm64
+      - Build_Linux_Arm
+      - Build_Linux_x64_Alpine36
+      # - Build_Linux_x64_FreeBSD
+      - Build_Linux_x64_Rhel6
+      - Build_Windows_x64
+      - Build_Windows_x86
+      - Build_Windows_Arm
+      - Build_Windows_Arm64
+      - Build_OSX
+    pool:
+      # For public or PR jobs, use the hosted pool.  For internal jobs use the internal pool.
+      # Will eventually change this to two BYOC pools.
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        name: Hosted VS2017
+      ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        name: dotnet-internal-temp
+    variables:
+      _PublishType: ${{ parameters._PublishType}}
+    steps:
+
+    # Initialize tooling
+    - script: build.cmd
+        -- /t:BuildTraversalBuildDependencies
+      displayName: Initialize tooling
+      condition: succeeded()
+
+    - task: MicroBuildSigningPlugin@2
+      displayName: Install MicroBuild plugin for Signing
+      inputs:
+        signType: $(SignType)
+        zipSources: false
+        feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
+      continueOnError: false
+      condition: and(succeeded(), in(variables['SignType'], 'real', 'test'))
+
+    - task: MSBuild@1
+      displayName: Publish (no PublishType)
+      inputs: 
+        solution: $(Build.SourcesDirectory)\publish\publish.proj
+        platform: x64
+        configuration: Release
+        msbuildVersion: 15.0
+        msbuildArchitecture: x64
+        msbuildArguments: '/p:Finalize=true 
+        /p:OfficialPublish=true 
+        /p:PublishType=nopublishtype 
+        /p:SignType=$(SignType) 
+        /p:StabilizePackageVersion=$(IsStable) 
+        /p:TargetArchitecture=x64
+        $(_BlobFeedArgs) 
+        $(_CommonPublishArgs) 
+        $(_NugetFeedArgs) 
+        $(_SymbolServerArgs) 
+        /bl:$(Build.SourcesDirectory)\finalizepublish.binlog'
+      condition: and(succeeded(), eq(variables._PublishType, 'nopublishtype'))
+
+    - task: CopyFiles@2
+      displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)'
+        Contents: |
+          *.log
+          *.binlog
+        TargetFolder: '$(Build.StagingDirectory)\BuildLogs'
+      continueOnError: true
+      condition: succeededOrFailed()
+
+    - task: CopyFiles@2
+      displayName: Copy Manifest file
+      inputs:
+        SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/AssetManifest'
+        TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
+      continueOnError: true
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Publish Artifact BuildLogs
+      inputs:
+        PathtoPublish: '$(Build.StagingDirectory)\BuildLogs'
+        ArtifactName: Finalize_Publish
+      condition: succeededOrFailed()
+
+    - task: PublishBuildArtifacts@1
+      displayName: Push Asset Manifests
+      inputs:
+        PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
+        PublishLocation: Container
+        ArtifactName: AssetManifests
+      continueOnError: true
+      condition: succeededOrFailed()